diff --git a/cartography/data/indexes.cypher b/cartography/data/indexes.cypher index 776e9bb7c6..872dd742d6 100644 --- a/cartography/data/indexes.cypher +++ b/cartography/data/indexes.cypher @@ -89,8 +89,6 @@ CREATE INDEX IF NOT EXISTS FOR (n:DOProject) ON (n.lastupdated); CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.id); CREATE INDEX IF NOT EXISTS FOR (n:EBSSnapshot) ON (n.lastupdated); CREATE INDEX IF NOT EXISTS FOR (n:EC2KeyPair) ON (n.keyfingerprint); -CREATE INDEX IF NOT EXISTS FOR (n:EC2PrivateIp) ON (n.id); -CREATE INDEX IF NOT EXISTS FOR (n:EC2PrivateIp) ON (n.lastupdated); CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.id); CREATE INDEX IF NOT EXISTS FOR (n:EC2ReservedInstance) ON (n.lastupdated); CREATE INDEX IF NOT EXISTS FOR (n:ECRImage) ON (n.id); diff --git a/cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json b/cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json deleted file mode 100644 index 9eed8cab1c..0000000000 --- a/cartography/data/jobs/cleanup/aws_ingest_network_interfaces_cleanup.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "statements": [ - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSVpc)<-[:MEMBER_OF_AWS_VPC]-(:EC2Subnet)<-[:PART_OF_SUBNET]-(:NetworkInterface)-[:PRIVATE_IP_ADDRESS]->(n:EC2PrivateIp) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)", - "iterative": true, - "iterationsize": 100 - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSVpc)<-[:MEMBER_OF_AWS_VPC]-(:EC2Subnet)<-[:PART_OF_SUBNET]-(:NetworkInterface)-[r:PRIVATE_IP_ADDRESS]->(:EC2PrivateIp) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)", - "iterative": true, - "iterationsize": 100 - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSVpc)<-[:MEMBER_OF_AWS_VPC]-(:EC2Subnet)<-[:PART_OF_SUBNET]-(n:NetworkInterface) WHERE n.lastupdated <> $UPDATE_TAG WITH n LIMIT $LIMIT_SIZE DETACH DELETE (n)", - "iterationsize": 100, - "iterative": true - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:AWSVpc)<-[:MEMBER_OF_AWS_VPC]-(:EC2Subnet)<-[r:PART_OF_SUBNET]-(:NetworkInterface) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)", - "iterationsize": 100, - "iterative": true - }, - { - "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(:LoadBalancer)-[r:PART_OF_SUBNET]->(:EC2Subnet) WHERE r.lastupdated <> $UPDATE_TAG WITH r LIMIT $LIMIT_SIZE DELETE (r)", - "iterationsize": 100, - "iterative": true - } - ], - "name": "cleanup NetworkInterface" -} diff --git a/cartography/intel/aws/ec2/instances.py b/cartography/intel/aws/ec2/instances.py index 87c7e32028..d288c27e63 100644 --- a/cartography/intel/aws/ec2/instances.py +++ b/cartography/intel/aws/ec2/instances.py @@ -13,10 +13,10 @@ from cartography.intel.aws.ec2.util import get_botocore_config from cartography.models.aws.ec2.instances import EC2InstanceSchema from cartography.models.aws.ec2.keypairs import EC2KeyPairSchema -from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceInstanceSchema from cartography.models.aws.ec2.reservations import EC2ReservationSchema -from cartography.models.aws.ec2.securitygroups import EC2SecurityGroupSchema -from cartography.models.aws.ec2.subnets import EC2SubnetSchema +from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema +from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.models.aws.ec2.volumes import EBSVolumeInstanceSchema from cartography.util import aws_handle_regions from cartography.util import timeit @@ -183,7 +183,7 @@ def load_ec2_subnets( ) -> None: load( neo4j_session, - EC2SubnetSchema(), + EC2SubnetInstanceSchema(), subnet_list, Region=region, AWS_ID=current_aws_account_id, @@ -219,7 +219,7 @@ def load_ec2_security_groups( ) -> None: load( neo4j_session, - EC2SecurityGroupSchema(), + EC2SecurityGroupInstanceSchema(), sg_list, Region=region, AWS_ID=current_aws_account_id, @@ -237,7 +237,7 @@ def load_ec2_network_interfaces( ) -> None: load( neo4j_session, - EC2NetworkInterfaceSchema(), + EC2NetworkInterfaceInstanceSchema(), network_interface_list, Region=region, AWS_ID=current_aws_account_id, diff --git a/cartography/intel/aws/ec2/network_interfaces.py b/cartography/intel/aws/ec2/network_interfaces.py index 612d93586a..f62496c3bc 100644 --- a/cartography/intel/aws/ec2/network_interfaces.py +++ b/cartography/intel/aws/ec2/network_interfaces.py @@ -1,5 +1,7 @@ import logging import re +from collections import namedtuple +from typing import Any from typing import Dict from typing import List @@ -7,18 +9,30 @@ import neo4j from .util import get_botocore_config +from cartography.client.core.tx import load from cartography.graph.job import GraphJob from cartography.models.aws.ec2.networkinterfaces import EC2NetworkInterfaceSchema +from cartography.models.aws.ec2.privateip_networkinterface import EC2PrivateIpNetworkInterfaceSchema +from cartography.models.aws.ec2.securitygroup_networkinterface import EC2SecurityGroupNetworkInterfaceSchema +from cartography.models.aws.ec2.subnet_networkinterface import EC2SubnetNetworkInterfaceSchema from cartography.util import aws_handle_regions -from cartography.util import run_cleanup_job from cartography.util import timeit logger = logging.getLogger(__name__) +Ec2NetworkData = namedtuple( + "Ec2NetworkData", [ + "network_interface_list", + "private_ip_list", + "sg_list", + "subnet_list", + ], +) + @timeit @aws_handle_regions -def get_network_interface_data(boto3_session: boto3.session.Session, region: str) -> List[Dict]: +def get_network_interface_data(boto3_session: boto3.session.Session, region: str) -> List[Dict[str, Any]]: client = boto3_session.client('ec2', region_name=region, config=get_botocore_config()) paginator = client.get_paginator('describe_network_interfaces') subnets: List[Dict] = [] @@ -27,256 +41,215 @@ def get_network_interface_data(boto3_session: boto3.session.Session, region: str return subnets -@timeit -def load_network_interfaces( - neo4j_session: neo4j.Session, data: Dict, region: str, aws_account_id: str, - update_tag: int, -) -> None: - """ - Creates (:NetworkInterface), - (:NetworkInterface)-[:RESOURCE]->(:AWSAccount), - (:NetworkInterface)-[:MEMBER_OF_EC2_SECURITY_GROUP]->(:EC2SecurityGroup), - (:NetworkInterface)-[:PART_OF_SUBNET]->(:EC2Subnet), - (:PrivateIpAddress), - (:NetworkInterface)-[:PRIVATE_IP_ADDRESS]->(:PrivateIpAddress) - """ - logger.debug("Loading %d network interfaces in %s.", len(data), region) - ingest_network_interfaces = """ - UNWIND $network_interfaces AS network_interface - MERGE (netinf:NetworkInterface{id: network_interface.NetworkInterfaceId}) - ON CREATE SET netinf.firstseen = timestamp() - SET netinf.lastupdated = $update_tag, - netinf.mac_address = network_interface.MacAddress, - netinf.description = network_interface.Description, - netinf.private_ip_address = network_interface.PrivateIpAddress, - netinf.id = network_interface.NetworkInterfaceId, - netinf.private_dns_name = network_interface.PrivateDnsName, - netinf.status = network_interface.Status, - netinf.subnetid = network_interface.SubnetId, - netinf.interface_type = network_interface.InterfaceType, - netinf.requester_managed = network_interface.RequesterManaged, - netinf.source_dest_check = network_interface.SourceDestCheck, - netinf.requester_id = network_interface.RequesterId, - netinf.public_ip = network_interface.Association.PublicIp - WITH network_interface, netinf +def transform_network_interface_data(data_list: List[Dict[str, Any]], region: str) -> Ec2NetworkData: + network_interface_list = [] + private_ip_list = [] + sg_list = [] + subnet_list = [] - UNWIND network_interface.PrivateIpAddresses AS private_ip_address - MERGE (private_ip:EC2PrivateIp{id: network_interface.NetworkInterfaceId + ':' - + private_ip_address.PrivateIpAddress}) - ON CREATE SET private_ip.firstseen = timestamp() - SET private_ip.lastupdated = $update_tag, - private_ip.network_interface_id = network_interface.NetworkInterfaceId, - private_ip.primary = private_ip_address.Primary, - private_ip.private_ip_address = private_ip_address.PrivateIpAddress, - private_ip.public_ip = private_ip_address.Association.PublicIp, - private_ip.ip_owner_id = private_ip_address.Association.IpOwnerId - - MERGE (netinf)-[r:PRIVATE_IP_ADDRESS]->(private_ip) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - WITH network_interface, netinf - - UNWIND network_interface.Groups AS security_group - MERGE (sg:EC2SecurityGroup{id: security_group.GroupId}) - ON CREATE SET sg.firstseen = timestamp() - SET sg.lastupdated = $update_tag - MERGE (netinf)-[r:MEMBER_OF_EC2_SECURITY_GROUP]->(sg) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - WITH network_interface, netinf - MERGE (acc:AWSAccount{id: $aws_account_id}) - ON CREATE SET acc.firstseen = timestamp(), acc.inscope=true - SET acc.lastupdated = $update_tag - MERGE (acc)-[r:RESOURCE]->(netinf) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - WITH network_interface, netinf - - MERGE (snet:EC2Subnet{subnetid: network_interface.SubnetId}) - ON CREATE SET snet.firstseen = timestamp() - SET snet.lastupdated = $update_tag - MERGE (netinf)-[r:PART_OF_SUBNET]->(snet) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - neo4j_session.run( - ingest_network_interfaces, network_interfaces=data, update_tag=update_tag, - region=region, aws_account_id=aws_account_id, + for network_interface in data_list: + # Parse network interface description for ELB association + # https://aws.amazon.com/premiumsupport/knowledge-center/elb-find-load-balancer-IP/ + elb_v1_id = None + elb_v2_id = None + elb_match = re.match(r'^ELB (?:net|app)/([^\/]+)\/(.*)', network_interface.get('Description', '')) + if elb_match: + elb_v1_id = f'{elb_match[1]}-{elb_match[2]}.elb.{region}.amazonaws.com', + else: + elb_match = re.match(r'^ELB (.*)', network_interface.get('Description', '')) + if elb_match: + elb_v2_id = elb_match[1] + # TODO issue #1024 change this to arn when ready + network_interface_id = network_interface['NetworkInterfaceId'] + network_interface_list.append( + { + 'Id': network_interface_id, + 'NetworkInterfaceId': network_interface['NetworkInterfaceId'], + 'Description': network_interface['Description'], + 'InstanceId': network_interface.get('Attachment', {}).get('InstanceId'), + 'InterfaceType': network_interface['InterfaceType'], + 'MacAddress': network_interface['MacAddress'], + 'PrivateDnsName': network_interface['PrivateDnsName'], + 'PrivateIpAddress': network_interface['PrivateIpAddress'], + 'PublicIp': network_interface.get('Association', {}).get('PublicIp'), + 'RequesterId': network_interface.get('RequesterId'), + 'RequesterManaged': network_interface['RequesterManaged'], + 'SourceDestCheck': network_interface['SourceDestCheck'], + 'Status': network_interface['Status'], + 'SubnetId': network_interface['SubnetId'], + 'ElbV1Id': elb_v1_id, + 'ElbV2Id': elb_v2_id, + }, + ) + if network_interface.get('PrivateIpAddresses'): + for private_ip_address in network_interface['PrivateIpAddresses']: + private_ip_list.append( + { + 'Id': f"{network_interface['NetworkInterfaceId']}:{private_ip_address['PrivateIpAddress']}", + 'NetworkInterfaceId': network_interface['NetworkInterfaceId'], + 'IpOwnerId': private_ip_address.get('Association', {}).get('IpOwnerId'), + 'Primary': private_ip_address['Primary'], + 'PrivateIpAddress': private_ip_address['PrivateIpAddress'], + 'PublicIp': private_ip_address.get('Association', {}).get('PublicIp'), + }, + ) + + if network_interface.get("Groups"): + for group in network_interface["Groups"]: + sg_list.append( + { + 'GroupId': group['GroupId'], + 'NetworkInterfaceId': network_interface_id, + }, + ) + + subnet_id = network_interface.get('SubnetId') + if subnet_id: + subnet_list.append( + { + 'NetworkInterfaceId': network_interface_id, + 'SubnetId': subnet_id, + 'ElbV1Id': elb_v1_id, + 'ElbV2Id': elb_v2_id, + }, + ) + + return Ec2NetworkData( + network_interface_list=network_interface_list, + private_ip_list=private_ip_list, + sg_list=sg_list, + subnet_list=subnet_list, ) @timeit -def load_network_interface_instance_relations( - neo4j_session: neo4j.Session, instance_associations: List[Dict], region: str, aws_account_id: str, update_tag: int, +def load_network_interfaces( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + region: str, + aws_account_id: str, + update_tag: int, ) -> None: - """ - Creates (:EC2Instance)-[:NETWORK_INTERFACE]->(:NetworkInterface) - """ - ingest_network_interface_instance_relations = """ - UNWIND $instance_associations AS instance_association - MATCH (netinf:NetworkInterface{id: instance_association.netinf_id}), - (instance:EC2Instance{id: instance_association.instance_id}) - MERGE (instance)-[r:NETWORK_INTERFACE]->(netinf) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("Attaching %d EC2 instances to network interfaces in %s.", len(instance_associations), region) - neo4j_session.run( - ingest_network_interface_instance_relations, instance_associations=instance_associations, - update_tag=update_tag, region=region, aws_account_id=aws_account_id, + logger.info(f"Loading {len(data)} network interfaces in {region}.") + load( + neo4j_session, + EC2NetworkInterfaceSchema(), + data, + Region=region, + AWS_ID=aws_account_id, + lastupdated=update_tag, ) @timeit -def load_network_interface_elb_relations( - neo4j_session: neo4j.Session, elb_associations: List[Dict], region: str, - aws_account_id: str, update_tag: int, +def load_private_ip_network_interface( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + region: str, + aws_account_id: str, + update_tag: int, ) -> None: """ - Creates (:LoadBalancer)-[:NETWORK_INTERFACE]->(:NetworkInterface) + Private IPs as known by describe-network-interfaces. """ - ingest_network_interface_elb_relations = """ - UNWIND $elb_associations AS elb_association - MATCH (netinf:NetworkInterface{id: elb_association.netinf_id}), - (elb:LoadBalancer{name: elb_association.elb_name}) - MERGE (elb)-[r:NETWORK_INTERFACE]->(netinf) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("Attaching %d ELBs to network interfaces in %s.", len(elb_associations), region) - neo4j_session.run( - ingest_network_interface_elb_relations, elb_associations=elb_associations, - update_tag=update_tag, region=region, aws_account_id=aws_account_id, + logger.info(f"Loading {len(data)} private IPs in {region}.") + load( + neo4j_session, + EC2PrivateIpNetworkInterfaceSchema(), + data, + Region=region, + AWS_ID=aws_account_id, + lastupdated=update_tag, ) @timeit -def load_network_interface_elbv2_relations( - neo4j_session: neo4j.Session, elb_associations_v2: List[Dict], region: str, - aws_account_id: str, update_tag: int, +def load_security_group_network_interface( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + region: str, + aws_account_id: str, + update_tag: int, ) -> None: """ - Creates (:LoadBalancerV2)-[:NETWORK_INTERFACE]->(:NetworkInterface) - """ - ingest_network_interface_elb2_relations = """ - UNWIND $elb_associations AS elb_association - MATCH (netinf:NetworkInterface{id: elb_association.netinf_id}), - (elb:LoadBalancerV2{id: elb_association.elb_id}) - MERGE (elb)-[r:NETWORK_INTERFACE]->(netinf) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag + Security groups as known by describe-network-interfaces. """ - logger.debug("Attaching %d ELB V2s to network interfaces in %s.", len(elb_associations_v2), region) - neo4j_session.run( - ingest_network_interface_elb2_relations, elb_associations=elb_associations_v2, - update_tag=update_tag, region=region, aws_account_id=aws_account_id, + logger.info(f"Loading {len(data)} security groups in {region}.") + load( + neo4j_session, + EC2SecurityGroupNetworkInterfaceSchema(), + data, + Region=region, + AWS_ID=aws_account_id, + lastupdated=update_tag, ) @timeit -def load_network_interface_instance_to_subnet_relations(neo4j_session: neo4j.Session, update_tag: int) -> None: - """ - Creates (:EC2Instance)-[:PART_OF_SUBNET]->(:EC2Subnet) if - (:EC2Instance)--(:NetworkInterface)--(:EC2Subnet). - """ - ingest_network_interface_instance_relations = """ - MATCH (i:EC2Instance)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) - MERGE (i)-[r:PART_OF_SUBNET]->(s) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("-> Instance to subnet") - neo4j_session.run( - ingest_network_interface_instance_relations, update_tag=update_tag, - ) - - -@timeit -def load_network_interface_load_balancer_relations(neo4j_session: neo4j.Session, update_tag: int) -> None: - """ - Creates (:LoadBalancer)-[:PART_OF_SUBNET]->(:EC2Subnet) if - (:LoadBalancer)--(:NetworkInterface)--(:EC2Subnet). - """ - ingest_network_interface_loadbalancer_relations = """ - MATCH (i:LoadBalancer)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) - MERGE (i)-[r:PART_OF_SUBNET]->(s) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag - """ - logger.debug("-> ELB to subnet") - neo4j_session.run( - ingest_network_interface_loadbalancer_relations, update_tag=update_tag, - ) - - -@timeit -def load_network_interface_load_balancer_v2_relations(neo4j_session: neo4j.Session, update_tag: int) -> None: - """ - Creates (:LoadBalancerV2)-[:PART_OF_SUBNET]->(:EC2Subnet) if - (:LoadBalancerV2)--(:NetworkInterface)--(:EC2Subnet). +def load_subnet_network_interface( + neo4j_session: neo4j.Session, + data: List[Dict[str, Any]], + region: str, + aws_account_id: str, + update_tag: int, +) -> None: """ - ingest_network_interface_loadbalancerv2_relations = """ - MATCH (i:LoadBalancerV2)-[:NETWORK_INTERFACE]-(interface:NetworkInterface)-[:PART_OF_SUBNET]-(s:EC2Subnet) - MERGE (i)-[r:PART_OF_SUBNET]->(s) - ON CREATE SET r.firstseen = timestamp() - SET r.lastupdated = $update_tag + Subnets as known by describe-network-interfaces. """ - logger.debug("-> ELBv2 to subnet") - neo4j_session.run( - ingest_network_interface_loadbalancerv2_relations, update_tag=update_tag, + logger.info(f"Loading {len(data)} subnets in {region}.") + load( + neo4j_session, + EC2SubnetNetworkInterfaceSchema(), + data, + Region=region, + AWS_ID=aws_account_id, + lastupdated=update_tag, ) -@timeit -def load(neo4j_session: neo4j.Session, data: List[Dict], region: str, aws_account_id: str, update_tag: int) -> None: - elb_associations = [] - elb_associations_v2 = [] - instance_associations = [] - - for network_interface in data: - # https://aws.amazon.com/premiumsupport/knowledge-center/elb-find-load-balancer-IP/ - matchObj = re.match(r'^ELB (?:net|app)/([^\/]+)\/(.*)', network_interface.get('Description', '')) - if matchObj: - elb_associations_v2.append({ - 'netinf_id': network_interface['NetworkInterfaceId'], - 'elb_id': f'{matchObj[1]}-{matchObj[2]}.elb.{region}.amazonaws.com', - }) - else: - matchObj = re.match(r'^ELB (.*)', network_interface.get('Description', '')) - if matchObj: - elb_associations.append({ - 'netinf_id': network_interface['NetworkInterfaceId'], - 'elb_name': matchObj[1], - }) - - if 'Attachment' in network_interface and 'InstanceId' in network_interface['Attachment']: - instance_associations.append({ - 'netinf_id': network_interface['NetworkInterfaceId'], - 'instance_id': network_interface['Attachment']['InstanceId'], - }) - load_network_interfaces(neo4j_session, data, region, aws_account_id, update_tag) # type: ignore - load_network_interface_instance_relations( - neo4j_session, instance_associations, region, aws_account_id, update_tag, - ) - load_network_interface_elb_relations(neo4j_session, elb_associations, region, aws_account_id, update_tag) - load_network_interface_elbv2_relations(neo4j_session, elb_associations_v2, region, aws_account_id, update_tag) - load_network_interface_instance_to_subnet_relations(neo4j_session, update_tag) - load_network_interface_load_balancer_relations(neo4j_session, update_tag) +def load_network_data( + neo4j_session: neo4j.Session, + region: str, + current_aws_account_id: str, + update_tag: int, + network_interface_list: List[Dict[str, Any]], + private_ip_list: List[Dict[str, Any]], + subnet_list: List[Dict[str, Any]], + sg_list: List[Dict[str, Any]], +) -> None: + load_network_interfaces(neo4j_session, network_interface_list, region, current_aws_account_id, update_tag) + load_private_ip_network_interface(neo4j_session, private_ip_list, region, current_aws_account_id, update_tag) + load_subnet_network_interface(neo4j_session, subnet_list, region, current_aws_account_id, update_tag) + load_security_group_network_interface(neo4j_session, sg_list, region, current_aws_account_id, update_tag) @timeit def cleanup_network_interfaces(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: - run_cleanup_job('aws_ingest_network_interfaces_cleanup.json', neo4j_session, common_job_parameters) GraphJob.from_node_schema(EC2NetworkInterfaceSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(EC2PrivateIpNetworkInterfaceSchema(), common_job_parameters).run(neo4j_session) @timeit def sync_network_interfaces( - neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str], current_aws_account_id: str, - update_tag: int, common_job_parameters: Dict, + neo4j_session: neo4j.Session, + boto3_session: boto3.session.Session, + regions: List[str], + current_aws_account_id: str, + update_tag: int, + common_job_parameters: Dict, ) -> None: for region in regions: - logger.info("Syncing EC2 network interfaces for region '%s' in account '%s'.", region, current_aws_account_id) + logger.info(f"Syncing EC2 network interfaces for region '{region}' in account '{current_aws_account_id}'.") data = get_network_interface_data(boto3_session, region) - load(neo4j_session, data, region, current_aws_account_id, update_tag) + ec2_network_data = transform_network_interface_data(data, region) + load_network_data( + neo4j_session, + region, + current_aws_account_id, + update_tag, + ec2_network_data.network_interface_list, + ec2_network_data.private_ip_list, + ec2_network_data.subnet_list, + ec2_network_data.sg_list, + ) cleanup_network_interfaces(neo4j_session, common_job_parameters) diff --git a/cartography/intel/aws/ec2/security_groups.py b/cartography/intel/aws/ec2/security_groups.py index 9819f6490c..c0a66c8d22 100644 --- a/cartography/intel/aws/ec2/security_groups.py +++ b/cartography/intel/aws/ec2/security_groups.py @@ -8,7 +8,7 @@ from .util import get_botocore_config from cartography.graph.job import GraphJob -from cartography.models.aws.ec2.securitygroups import EC2SecurityGroupSchema +from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -148,7 +148,7 @@ def cleanup_ec2_security_groupinfo(neo4j_session: neo4j.Session, common_job_para neo4j_session, common_job_parameters, ) - GraphJob.from_node_schema(EC2SecurityGroupSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(EC2SecurityGroupInstanceSchema(), common_job_parameters).run(neo4j_session) @timeit diff --git a/cartography/intel/aws/ec2/subnets.py b/cartography/intel/aws/ec2/subnets.py index a46b39c141..d306049835 100644 --- a/cartography/intel/aws/ec2/subnets.py +++ b/cartography/intel/aws/ec2/subnets.py @@ -7,7 +7,7 @@ from .util import get_botocore_config from cartography.graph.job import GraphJob -from cartography.models.aws.ec2.subnets import EC2SubnetSchema +from cartography.models.aws.ec2.subnet_instance import EC2SubnetInstanceSchema from cartography.util import aws_handle_regions from cartography.util import run_cleanup_job from cartography.util import timeit @@ -78,7 +78,7 @@ def load_subnets( @timeit def cleanup_subnets(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None: run_cleanup_job('aws_ingest_subnets_cleanup.json', neo4j_session, common_job_parameters) - GraphJob.from_node_schema(EC2SubnetSchema(), common_job_parameters).run(neo4j_session) + GraphJob.from_node_schema(EC2SubnetInstanceSchema(), common_job_parameters).run(neo4j_session) @timeit diff --git a/cartography/models/aws/ec2/loadbalancerv2.py b/cartography/models/aws/ec2/loadbalancerv2.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cartography/models/aws/ec2/networkinterface_instance.py b/cartography/models/aws/ec2/networkinterface_instance.py new file mode 100644 index 0000000000..cb1e2476d4 --- /dev/null +++ b/cartography/models/aws/ec2/networkinterface_instance.py @@ -0,0 +1,109 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class EC2NetworkInterfaceInstanceNodeProperties(CartographyNodeProperties): + """ + Selection of properties of a network interface as known by an EC2 instance + """ + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO use arn; issue #1024 + id: PropertyRef = PropertyRef('NetworkInterfaceId') + status: PropertyRef = PropertyRef('Status') + mac_address: PropertyRef = PropertyRef('MacAddress', extra_index=True) + description: PropertyRef = PropertyRef('Description') + private_dns_name: PropertyRef = PropertyRef('PrivateDnsName', extra_index=True) + private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress', extra_index=True) + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToAwsAccountRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToAWSAccount(CartographyRelSchema): + target_node_label: str = 'AWSAccount' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: EC2NetworkInterfaceToAwsAccountRelProperties = EC2NetworkInterfaceToAwsAccountRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2InstanceRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2Instance(CartographyRelSchema): + target_node_label: str = 'EC2Instance' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('InstanceId')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "NETWORK_INTERFACE" + properties: EC2NetworkInterfaceToEC2InstanceRelProperties = EC2NetworkInterfaceToEC2InstanceRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2SubnetRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2Subnet(CartographyRelSchema): + target_node_label: str = 'EC2Subnet' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('SubnetId')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "PART_OF_SUBNET" + properties: EC2NetworkInterfaceToEC2SubnetRelProperties = EC2NetworkInterfaceToEC2SubnetRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2SecurityGroupRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToEC2SecurityGroup(CartographyRelSchema): + target_node_label: str = 'EC2SecurityGroup' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('GroupId')}, + ) + direction: LinkDirection = LinkDirection.OUTWARD + rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP" + properties: EC2NetworkInterfaceToEC2SecurityGroupRelProperties = \ + EC2NetworkInterfaceToEC2SecurityGroupRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceInstanceSchema(CartographyNodeSchema): + """ + Network interface as known by an EC2 instance + """ + label: str = 'NetworkInterface' + properties: EC2NetworkInterfaceInstanceNodeProperties = EC2NetworkInterfaceInstanceNodeProperties() + sub_resource_relationship: EC2NetworkInterfaceToAWSAccount = EC2NetworkInterfaceToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2NetworkInterfaceToEC2Instance(), + EC2NetworkInterfaceToEC2Subnet(), + EC2NetworkInterfaceToEC2SecurityGroup(), + ], + ) diff --git a/cartography/models/aws/ec2/networkinterfaces.py b/cartography/models/aws/ec2/networkinterfaces.py index 3c536407ce..a21d4ba859 100644 --- a/cartography/models/aws/ec2/networkinterfaces.py +++ b/cartography/models/aws/ec2/networkinterfaces.py @@ -1,5 +1,9 @@ from dataclasses import dataclass +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToAWSAccount +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2Instance +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2SecurityGroup +from cartography.models.aws.ec2.networkinterface_instance import EC2NetworkInterfaceToEC2Subnet from cartography.models.core.common import PropertyRef from cartography.models.core.nodes import CartographyNodeProperties from cartography.models.core.nodes import CartographyNodeSchema @@ -13,90 +17,73 @@ @dataclass(frozen=True) class EC2NetworkInterfaceNodeProperties(CartographyNodeProperties): - # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO decide this + """ + Network interface properties + """ id: PropertyRef = PropertyRef('NetworkInterfaceId') - status: PropertyRef = PropertyRef('Status') - mac_address: PropertyRef = PropertyRef('MacAddress') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) description: PropertyRef = PropertyRef('Description') + mac_address: PropertyRef = PropertyRef('MacAddress', extra_index=True) private_dns_name: PropertyRef = PropertyRef('PrivateDnsName') - private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress') + private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress', extra_index=True) region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) - lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) - - -@dataclass(frozen=True) -class EC2NetworkInterfaceToAwsAccountRelProperties(CartographyRelProperties): - lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) - + status: PropertyRef = PropertyRef('Status') -@dataclass(frozen=True) -class EC2NetworkInterfaceToAWSAccount(CartographyRelSchema): - target_node_label: str = 'AWSAccount' - target_node_matcher: TargetNodeMatcher = make_target_node_matcher( - {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, - ) - direction: LinkDirection = LinkDirection.INWARD - rel_label: str = "RESOURCE" - properties: EC2NetworkInterfaceToAwsAccountRelProperties = EC2NetworkInterfaceToAwsAccountRelProperties() + # Properties only returned by describe-network-interfaces + interface_type: PropertyRef = PropertyRef('InterfaceType') + public_ip: PropertyRef = PropertyRef('PublicIp', extra_index=True) + requester_id: PropertyRef = PropertyRef('RequesterId', extra_index=True) + requester_managed: PropertyRef = PropertyRef('RequesterManaged') + source_dest_check: PropertyRef = PropertyRef('SourceDestCheck') + subnetid: PropertyRef = PropertyRef('SubnetId', extra_index=True) @dataclass(frozen=True) -class EC2NetworkInterfaceToEC2InstanceRelProperties(CartographyRelProperties): +class EC2NetworkInterfaceToElbRelProperties(CartographyRelProperties): lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) @dataclass(frozen=True) -class EC2NetworkInterfaceToEC2Instance(CartographyRelSchema): - target_node_label: str = 'EC2Instance' +class EC2NetworkInterfaceToElb(CartographyRelSchema): + target_node_label: str = 'LoadBalancer' target_node_matcher: TargetNodeMatcher = make_target_node_matcher( - {'id': PropertyRef('InstanceId')}, + {'name': PropertyRef('ElbV1Id')}, ) direction: LinkDirection = LinkDirection.INWARD rel_label: str = "NETWORK_INTERFACE" - properties: EC2NetworkInterfaceToEC2InstanceRelProperties = EC2NetworkInterfaceToEC2InstanceRelProperties() + properties: EC2NetworkInterfaceToElbRelProperties = EC2NetworkInterfaceToElbRelProperties() @dataclass(frozen=True) -class EC2NetworkInterfaceToEC2SubnetRelProperties(CartographyRelProperties): +class EC2NetworkInterfaceToElbV2RelProperties(CartographyRelProperties): lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) @dataclass(frozen=True) -class EC2NetworkInterfaceToEC2Subnet(CartographyRelSchema): - target_node_label: str = 'EC2Subnet' +class EC2NetworkInterfaceToElbV2(CartographyRelSchema): + target_node_label: str = 'LoadBalancerV2' target_node_matcher: TargetNodeMatcher = make_target_node_matcher( - {'id': PropertyRef('SubnetId')}, + {'id': PropertyRef('ElbV2Id')}, ) - direction: LinkDirection = LinkDirection.OUTWARD - rel_label: str = "PART_OF_SUBNET" - properties: EC2NetworkInterfaceToEC2SubnetRelProperties = EC2NetworkInterfaceToEC2SubnetRelProperties() - - -@dataclass(frozen=True) -class EC2NetworkInterfaceToEC2SecurityGroupRelProperties(CartographyRelProperties): - lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) - - -@dataclass(frozen=True) -class EC2NetworkInterfaceToEC2SecurityGroup(CartographyRelSchema): - target_node_label: str = 'EC2SecurityGroup' - target_node_matcher: TargetNodeMatcher = make_target_node_matcher( - {'id': PropertyRef('GroupId')}, - ) - direction: LinkDirection = LinkDirection.OUTWARD - rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP" - properties: EC2NetworkInterfaceToEC2SubnetRelProperties = EC2NetworkInterfaceToEC2SubnetRelProperties() + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "NETWORK_INTERFACE" + properties: EC2NetworkInterfaceToElbV2RelProperties = EC2NetworkInterfaceToElbV2RelProperties() @dataclass(frozen=True) class EC2NetworkInterfaceSchema(CartographyNodeSchema): + """ + Network interface as known by describe-network-interfaces. + """ label: str = 'NetworkInterface' properties: EC2NetworkInterfaceNodeProperties = EC2NetworkInterfaceNodeProperties() sub_resource_relationship: EC2NetworkInterfaceToAWSAccount = EC2NetworkInterfaceToAWSAccount() other_relationships: OtherRelationships = OtherRelationships( [ - EC2NetworkInterfaceToEC2Instance(), EC2NetworkInterfaceToEC2Subnet(), EC2NetworkInterfaceToEC2SecurityGroup(), + EC2NetworkInterfaceToElb(), + EC2NetworkInterfaceToElbV2(), + EC2NetworkInterfaceToEC2Instance(), ], ) diff --git a/cartography/models/aws/ec2/privateip_networkinterface.py b/cartography/models/aws/ec2/privateip_networkinterface.py new file mode 100644 index 0000000000..1200520fbc --- /dev/null +++ b/cartography/models/aws/ec2/privateip_networkinterface.py @@ -0,0 +1,72 @@ +from dataclasses import dataclass + +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class EC2PrivateIpNetworkInterfaceNodeProperties(CartographyNodeProperties): + """ + Selection of properties of a private IP as known by an EC2 network interface + """ + id: PropertyRef = PropertyRef('Id') + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + network_interface_id: PropertyRef = PropertyRef('NetworkInterfaceId') + primary: PropertyRef = PropertyRef('Primary') + private_ip_address: PropertyRef = PropertyRef('PrivateIpAddress') + public_ip: PropertyRef = PropertyRef('PublicIp') + ip_owner_id: PropertyRef = PropertyRef('IpOwnerId') + + +@dataclass(frozen=True) +class EC2PrivateIpToAwsAccountRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2PrivateIpToAWSAccount(CartographyRelSchema): + target_node_label: str = 'AWSAccount' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('AWS_ID', set_in_kwargs=True)}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "RESOURCE" + properties: EC2PrivateIpToAwsAccountRelProperties = EC2PrivateIpToAwsAccountRelProperties() + + +@dataclass(frozen=True) +class EC2NetworkInterfaceToPrivateIpRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2PrivateIpToNetworkInterface(CartographyRelSchema): + target_node_label: str = 'NetworkInterface' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('NetworkInterfaceId')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "PRIVATE_IP_ADDRESS" + properties: EC2NetworkInterfaceToPrivateIpRelProperties = EC2NetworkInterfaceToPrivateIpRelProperties() + + +@dataclass(frozen=True) +class EC2PrivateIpNetworkInterfaceSchema(CartographyNodeSchema): + """ + PrivateIp as known by a Network Interface + """ + label: str = 'EC2PrivateIp' + properties: EC2PrivateIpNetworkInterfaceNodeProperties = EC2PrivateIpNetworkInterfaceNodeProperties() + sub_resource_relationship: EC2PrivateIpToAWSAccount = EC2PrivateIpToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2PrivateIpToNetworkInterface(), + ], + ) diff --git a/cartography/models/aws/ec2/securitygroups.py b/cartography/models/aws/ec2/securitygroup_instance.py similarity index 82% rename from cartography/models/aws/ec2/securitygroups.py rename to cartography/models/aws/ec2/securitygroup_instance.py index 34813340c7..041085d61e 100644 --- a/cartography/models/aws/ec2/securitygroups.py +++ b/cartography/models/aws/ec2/securitygroup_instance.py @@ -12,8 +12,8 @@ @dataclass(frozen=True) -class EC2SecurityGroupNodeProperties(CartographyNodeProperties): - # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO decide on this +class EC2SecurityGroupInstanceNodeProperties(CartographyNodeProperties): + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO use arn; #1024 id: PropertyRef = PropertyRef('GroupId') groupid: PropertyRef = PropertyRef('GroupId', extra_index=True) region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) @@ -37,7 +37,7 @@ class EC2SecurityGroupToAWSAccount(CartographyRelSchema): @dataclass(frozen=True) -class EC2SubnetToEC2InstanceRelProperties(CartographyRelProperties): +class EC2SecurityGroupToEC2InstanceRelProperties(CartographyRelProperties): lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) @@ -49,13 +49,16 @@ class EC2SecurityGroupToEC2Instance(CartographyRelSchema): ) direction: LinkDirection = LinkDirection.INWARD rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP" - properties: EC2SubnetToEC2InstanceRelProperties = EC2SubnetToEC2InstanceRelProperties() + properties: EC2SecurityGroupToEC2InstanceRelProperties = EC2SecurityGroupToEC2InstanceRelProperties() @dataclass(frozen=True) -class EC2SecurityGroupSchema(CartographyNodeSchema): +class EC2SecurityGroupInstanceSchema(CartographyNodeSchema): + """ + Security groups as known by describe-ec2-instances + """ label: str = 'EC2SecurityGroup' - properties: EC2SecurityGroupNodeProperties = EC2SecurityGroupNodeProperties() + properties: EC2SecurityGroupInstanceNodeProperties = EC2SecurityGroupInstanceNodeProperties() sub_resource_relationship: EC2SecurityGroupToAWSAccount = EC2SecurityGroupToAWSAccount() other_relationships: OtherRelationships = OtherRelationships( [ diff --git a/cartography/models/aws/ec2/securitygroup_networkinterface.py b/cartography/models/aws/ec2/securitygroup_networkinterface.py new file mode 100644 index 0000000000..01178f9eed --- /dev/null +++ b/cartography/models/aws/ec2/securitygroup_networkinterface.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass + +from cartography.models.aws.ec2.securitygroup_instance import EC2SecurityGroupToAWSAccount +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class EC2SecurityGroupNetworkInterfaceNodeProperties(CartographyNodeProperties): + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) # TODO use arn; issue #1024 + id: PropertyRef = PropertyRef('GroupId') + groupid: PropertyRef = PropertyRef('GroupId', extra_index=True) + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToNetworkInterfaceRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SecurityGroupToNetworkInterface(CartographyRelSchema): + target_node_label: str = 'NetworkInterface' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('NetworkInterfaceId')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "MEMBER_OF_EC2_SECURITY_GROUP" + properties: EC2SubnetToNetworkInterfaceRelProperties = EC2SubnetToNetworkInterfaceRelProperties() + + +@dataclass(frozen=True) +class EC2SecurityGroupNetworkInterfaceSchema(CartographyNodeSchema): + """ + Security groups as known by describe-network-interfaces. + """ + label: str = 'EC2SecurityGroup' + properties: EC2SecurityGroupNetworkInterfaceNodeProperties = EC2SecurityGroupNetworkInterfaceNodeProperties() + sub_resource_relationship: EC2SecurityGroupToAWSAccount = EC2SecurityGroupToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2SecurityGroupToNetworkInterface(), + ], + ) diff --git a/cartography/models/aws/ec2/subnets.py b/cartography/models/aws/ec2/subnet_instance.py similarity index 89% rename from cartography/models/aws/ec2/subnets.py rename to cartography/models/aws/ec2/subnet_instance.py index 62db69909f..9fe8040874 100644 --- a/cartography/models/aws/ec2/subnets.py +++ b/cartography/models/aws/ec2/subnet_instance.py @@ -12,8 +12,8 @@ @dataclass(frozen=True) -class EC2SubnetNodeProperties(CartographyNodeProperties): - # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO decide this +class EC2SubnetInstanceNodeProperties(CartographyNodeProperties): + # arn: PropertyRef = PropertyRef('Arn', extra_index=True) TODO use arn; issue #1024 id: PropertyRef = PropertyRef('SubnetId') subnet_id: PropertyRef = PropertyRef('SubnetId', extra_index=True) region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) @@ -53,9 +53,12 @@ class EC2SubnetToEC2Instance(CartographyRelSchema): @dataclass(frozen=True) -class EC2SubnetSchema(CartographyNodeSchema): +class EC2SubnetInstanceSchema(CartographyNodeSchema): + """ + EC2 Subnet as known by describe-ec2-instances + """ label: str = 'EC2Subnet' - properties: EC2SubnetNodeProperties = EC2SubnetNodeProperties() + properties: EC2SubnetInstanceNodeProperties = EC2SubnetInstanceNodeProperties() sub_resource_relationship: EC2SubnetToAWSAccount = EC2SubnetToAWSAccount() other_relationships: OtherRelationships = OtherRelationships( [ diff --git a/cartography/models/aws/ec2/subnet_networkinterface.py b/cartography/models/aws/ec2/subnet_networkinterface.py new file mode 100644 index 0000000000..8a1adc89e9 --- /dev/null +++ b/cartography/models/aws/ec2/subnet_networkinterface.py @@ -0,0 +1,87 @@ +from dataclasses import dataclass + +from cartography.models.aws.ec2.subnet_instance import EC2SubnetToAWSAccount +from cartography.models.aws.ec2.subnet_instance import EC2SubnetToEC2Instance +from cartography.models.core.common import PropertyRef +from cartography.models.core.nodes import CartographyNodeProperties +from cartography.models.core.nodes import CartographyNodeSchema +from cartography.models.core.relationships import CartographyRelProperties +from cartography.models.core.relationships import CartographyRelSchema +from cartography.models.core.relationships import LinkDirection +from cartography.models.core.relationships import make_target_node_matcher +from cartography.models.core.relationships import OtherRelationships +from cartography.models.core.relationships import TargetNodeMatcher + + +@dataclass(frozen=True) +class EC2SubnetNetworkInterfaceNodeProperties(CartographyNodeProperties): + id: PropertyRef = PropertyRef('SubnetId') + subnet_id: PropertyRef = PropertyRef('SubnetId', extra_index=True) + region: PropertyRef = PropertyRef('Region', set_in_kwargs=True) + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToNetworkInterfaceRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToNetworkInterface(CartographyRelSchema): + target_node_label: str = 'NetworkInterface' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('NetworkInterfaceId')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "PART_OF_SUBNET" + properties: EC2SubnetToNetworkInterfaceRelProperties = EC2SubnetToNetworkInterfaceRelProperties() + + +@dataclass(frozen=True) +class EC2SubnetToLoadBalancerRelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToLoadBalancer(CartographyRelSchema): + target_node_label: str = 'LoadBalancer' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('ElbV1Id')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "PART_OF_SUBNET" + properties: EC2SubnetToLoadBalancerRelProperties = EC2SubnetToLoadBalancerRelProperties() + + +@dataclass(frozen=True) +class EC2SubnetToLoadBalancerV2RelProperties(CartographyRelProperties): + lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True) + + +@dataclass(frozen=True) +class EC2SubnetToLoadBalancerV2(CartographyRelSchema): + target_node_label: str = 'LoadBalancerV2' + target_node_matcher: TargetNodeMatcher = make_target_node_matcher( + {'id': PropertyRef('ElbV2Id')}, + ) + direction: LinkDirection = LinkDirection.INWARD + rel_label: str = "PART_OF_SUBNET" + properties: EC2SubnetToLoadBalancerV2RelProperties = EC2SubnetToLoadBalancerV2RelProperties() + + +@dataclass(frozen=True) +class EC2SubnetNetworkInterfaceSchema(CartographyNodeSchema): + """ + Subnet as known by describe-network-interfaces + """ + label: str = 'EC2Subnet' + properties: EC2SubnetNetworkInterfaceNodeProperties = EC2SubnetNetworkInterfaceNodeProperties() + sub_resource_relationship: EC2SubnetToAWSAccount = EC2SubnetToAWSAccount() + other_relationships: OtherRelationships = OtherRelationships( + [ + EC2SubnetToNetworkInterface(), + EC2SubnetToEC2Instance(), + EC2SubnetToLoadBalancer(), + EC2SubnetToLoadBalancerV2(), + ], + ) diff --git a/cartography/util.py b/cartography/util.py index 30b9bccd80..144f17fb3e 100644 --- a/cartography/util.py +++ b/cartography/util.py @@ -49,6 +49,14 @@ def run_analysis_job( common_job_parameters: Dict, package: str = 'cartography.data.jobs.analysis', ) -> None: + """ + Enriches existing graph data with analysis jobs. This is designed for use with the sync stage + cartography.intel.analysis. + Runs the queries in the given Python `package` directory (cartography.data.jobs.analysis by default) for the given + `filename`. All queries in this directory are intended to be run at the end of a full graph sync. As such, they are + not scoped to a single sub resource. That is they will apply to _all_ AWS accounts/_all_ GCP projects/_all_ Okta + organizations/etc. + """ GraphJob.run_from_json( neo4j_session, read_text( diff --git a/docs/root/usage/tutorial.md b/docs/root/usage/tutorial.md index f0cb52fa9b..731285a393 100644 --- a/docs/root/usage/tutorial.md +++ b/docs/root/usage/tutorial.md @@ -150,7 +150,7 @@ If you want to learn more in depth about Neo4j and Cypher queries you can look a .. _data-augmentation: -Cartography adds custom attributes to nodes and relationships to point out security-related items of interest. Unless mentioned otherwise these data augmentation jobs are stored in `cartography/data/jobs/analysis`. Here is a summary of all of Cartography's custom attributes. +Cartography adds custom attributes to nodes and relationships to point out security-related items of interest. Data augmentation jobs meant to apply to the whole graph and run at the end of a sync are stored in `cartography/data/jobs/analysis`. Here is a summary of all of Cartography's custom attributes. - `exposed_internet` indicates whether the asset is accessible to the public internet. diff --git a/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py b/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py index 1c5d271ad8..2780f3f78d 100644 --- a/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py +++ b/tests/integration/cartography/intel/aws/ec2/test_ec2_network_interfaces.py @@ -1,122 +1,111 @@ -import cartography.intel.aws.ec2 -import tests.data.aws.ec2.network_interfaces +from unittest.mock import MagicMock +from unittest.mock import patch +import cartography.intel.aws.ec2.network_interfaces +from cartography.intel.aws.ec2.network_interfaces import sync_network_interfaces +from tests.data.aws.ec2.network_interfaces import DESCRIBE_NETWORK_INTERFACES +from tests.integration.cartography.intel.aws.common import create_test_account +from tests.integration.util import check_nodes +from tests.integration.util import check_rels TEST_ACCOUNT_ID = '000000000000' TEST_REGION = 'eu-north-1' TEST_UPDATE_TAG = 123456789 -def test_load_network_interfaces(neo4j_session): - data = tests.data.aws.ec2.network_interfaces.DESCRIBE_NETWORK_INTERFACES - cartography.intel.aws.ec2.network_interfaces.load( - neo4j_session, - data, - TEST_REGION, - TEST_ACCOUNT_ID, - TEST_UPDATE_TAG, - ) - - expected_nodes = { - "eni-0e106a07c15ff7d14", - "eni-0d9877f559c240362", - "eni-04b4289e1be7634e4", - } - - nodes = neo4j_session.run( - """ - MATCH (ni:NetworkInterface) RETURN ni.id; - """, - ) - actual_nodes = {n['ni.id'] for n in nodes} +@patch.object( + cartography.intel.aws.ec2.network_interfaces, + 'get_network_interface_data', + return_value=DESCRIBE_NETWORK_INTERFACES, +) +def test_load_network_interfaces(mock_get_network_interfaces, neo4j_session): + # Arrange + boto3_session = MagicMock() + create_test_account(neo4j_session, TEST_ACCOUNT_ID, TEST_UPDATE_TAG) - assert actual_nodes == expected_nodes - - -def test_ec2_private_ips(neo4j_session): - data = tests.data.aws.ec2.network_interfaces.DESCRIBE_NETWORK_INTERFACES - cartography.intel.aws.ec2.network_interfaces.load( + # Act + sync_network_interfaces( neo4j_session, - data, - TEST_REGION, + boto3_session, + [TEST_REGION], TEST_ACCOUNT_ID, TEST_UPDATE_TAG, + {'UPDATE_TAG': TEST_UPDATE_TAG, 'AWS_ID': TEST_ACCOUNT_ID}, ) - expected_nodes = { - "eni-0e106a07c15ff7d14:10.0.4.180", - "eni-0d9877f559c240362:10.0.4.96", - "eni-04b4289e1be7634e4:10.0.4.5", - "eni-04b4289e1be7634e4:10.0.4.12", + # Assert NetworkInterfaces were created + assert check_nodes(neo4j_session, 'NetworkInterface', ['id']) == { + ("eni-0e106a07c15ff7d14",), + ("eni-0d9877f559c240362",), + ("eni-04b4289e1be7634e4",), } - nodes = neo4j_session.run( - """ - MATCH (ni:EC2PrivateIp) RETURN ni.id; - """, - ) - actual_nodes = {n['ni.id'] for n in nodes} - - assert actual_nodes == expected_nodes - + # Assert EC2PrivateIps were created + assert check_nodes(neo4j_session, 'EC2PrivateIp', ['id']) == { + ("eni-0e106a07c15ff7d14:10.0.4.180",), + ("eni-0d9877f559c240362:10.0.4.96",), + ("eni-04b4289e1be7634e4:10.0.4.5",), + ("eni-04b4289e1be7634e4:10.0.4.12",), + } -def test_network_interface_relationships(neo4j_session): - data = tests.data.aws.ec2.network_interfaces.DESCRIBE_NETWORK_INTERFACES - cartography.intel.aws.ec2.network_interfaces.load( + # Assert NetworkInterface to PrivateIp rels exist + assert check_rels( neo4j_session, - data, - TEST_REGION, - TEST_ACCOUNT_ID, - TEST_UPDATE_TAG, - ) - - expected_nodes = { + 'NetworkInterface', + 'id', + 'EC2PrivateIp', + 'id', + 'PRIVATE_IP_ADDRESS', + rel_direction_right=True, + ) == { ('eni-0e106a07c15ff7d14', 'eni-0e106a07c15ff7d14:10.0.4.180'), ('eni-0d9877f559c240362', 'eni-0d9877f559c240362:10.0.4.96'), ('eni-04b4289e1be7634e4', 'eni-04b4289e1be7634e4:10.0.4.5'), ('eni-04b4289e1be7634e4', 'eni-04b4289e1be7634e4:10.0.4.12'), } - # Fetch relationships - result = neo4j_session.run( - """ - MATCH (n1:NetworkInterface)-[:PRIVATE_IP_ADDRESS]->(n2:EC2PrivateIp) RETURN n1.id, n2.id; - """, - ) - actual = { - (r['n1.id'], r['n2.id']) for r in result - } - - assert actual == expected_nodes - - -def test_network_interface_to_account(neo4j_session): - neo4j_session.run('MERGE (:AWSAccount{id:$ACC_ID})', ACC_ID=TEST_ACCOUNT_ID) - - data = tests.data.aws.ec2.network_interfaces.DESCRIBE_NETWORK_INTERFACES - cartography.intel.aws.ec2.network_interfaces.load( + # Assert NetworkInterface to AWSAccount rels exist + assert check_rels( neo4j_session, - data, - TEST_REGION, - TEST_ACCOUNT_ID, - TEST_UPDATE_TAG, - ) - - expected_nodes = { + 'NetworkInterface', + 'id', + 'AWSAccount', + 'id', + 'RESOURCE', + rel_direction_right=False, + ) == { ('eni-0e106a07c15ff7d14', TEST_ACCOUNT_ID), ('eni-0d9877f559c240362', TEST_ACCOUNT_ID), ('eni-04b4289e1be7634e4', TEST_ACCOUNT_ID), ('eni-04b4289e1be7634e4', TEST_ACCOUNT_ID), } - # Fetch relationships - result = neo4j_session.run( - """ - MATCH (n1:NetworkInterface)<-[:RESOURCE]-(n2:AWSAccount) RETURN n1.id, n2.id; - """, - ) - actual = { - (r['n1.id'], r['n2.id']) for r in result + # Assert NetworkInterface to Subnet rels exist + assert check_rels( + neo4j_session, + 'NetworkInterface', + 'id', + 'EC2Subnet', + 'id', + 'PART_OF_SUBNET', + rel_direction_right=True, + ) == { + ('eni-04b4289e1be7634e4', 'subnet-0fa10e76eeb24dbe7'), + ('eni-0d9877f559c240362', 'subnet-0fa10e76eeb24dbe7'), + ('eni-0e106a07c15ff7d14', 'subnet-0fa10e76eeb24dbe7'), } - assert actual == expected_nodes + # Assert NetworkInterface to security group rels exist + assert check_rels( + neo4j_session, + 'NetworkInterface', + 'id', + 'EC2SecurityGroup', + 'id', + 'MEMBER_OF_EC2_SECURITY_GROUP', + rel_direction_right=True, + ) == { + ('eni-04b4289e1be7634e4', 'sg-0e866e64db0c84705'), + ('eni-0d9877f559c240362', 'sg-0e866e64db0c84705'), + ('eni-0e106a07c15ff7d14', 'sg-0e866e64db0c84705'), + }