diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 1698be923b..2307bf3d7e 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -10,6 +10,10 @@ SREApplicationGatewayComponent, SREApplicationGatewayProps, ) +from .sre.apps import ( + SREAppsComponent, + SREAppsProps, +) from .sre.apt_proxy_server import SREAptProxyServerComponent, SREAptProxyServerProps from .sre.backup import ( SREBackupComponent, @@ -327,6 +331,17 @@ def __call__(self) -> None: tags=self.tags, ) + # Deploy apps + SREAppsComponent( + "sre_apps", + self.stack_name, + SREAppsProps( + location=self.config.azure.location, + resource_group_name=resource_group.name, + ), + tags=self.tags, + ) + # Deploy monitoring monitoring = SREMonitoringComponent( "sre_monitoring", diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py new file mode 100644 index 0000000000..40b6346fe4 --- /dev/null +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -0,0 +1,196 @@ +"""Pulumi component for SRE function/web apps""" + +from collections.abc import Mapping + +from pulumi import ComponentResource, FileArchive, Input, Output, ResourceOptions +from pulumi_azure_native import ( + storage, + web, +) + +from data_safe_haven.functions import ( + alphanumeric, + truncate_tokens, +) +from data_safe_haven.resources import resources_path + + +class SREAppsProps: + """Properties for SREAppsComponent""" + + def __init__( + self, + location: Input[str], + resource_group_name: Input[str], + ): + self.location = location + self.resource_group_name = resource_group_name + + +class SREAppsComponent(ComponentResource): + """Deploy SRE function/web apps with Pulumi""" + + def __init__( + self, + name: str, + stack_name: str, + props: SREAppsProps, + opts: ResourceOptions | None = None, + tags: Input[Mapping[str, Input[str]]] | None = None, + ) -> None: + super().__init__("dsh:sre:AppsComponent", name, {}, opts) + child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) + child_tags = tags if tags else {} + + # Deploy storage account + # The storage account holds app data/configuration + storage_account = storage.StorageAccount( + f"{self._name}_storage_account", + account_name=alphanumeric( + f"{''.join(truncate_tokens(stack_name.split('-'), 14))}apps" + )[:24], + kind=storage.Kind.STORAGE_V2, + location=props.location, + resource_group_name=props.resource_group_name, + sku=storage.SkuArgs(name=storage.SkuName.STANDARD_GRS), + opts=child_opts, + tags=child_tags, + ) + + # Create function apps container + container = storage.BlobContainer( + f"{self._name}_container_functions", + account_name=storage_account.name, + container_name="functions", + public_access=storage.PublicAccess.NONE, + resource_group_name=props.resource_group_name, + opts=ResourceOptions.merge( + child_opts, + ResourceOptions(parent=storage_account), + ), + ) + + # Upload Gitea mirror function app + blob_gitea_mirror = storage.Blob( + f"{self._name}_blob_gitea_mirror", + account_name=storage_account.name, + container_name=container.name, + resource_group_name=props.resource_group_name, + source=FileArchive( + str((resources_path / "gitea_mirror" / "functions").absolute()), + ), + opts=ResourceOptions.merge( + child_opts, + ResourceOptions(parent=container), + ), + ) + + # Get URL of app blob + blob_url = get_blob_url( + blob=blob_gitea_mirror, + container=container, + storage_account=storage_account, + resource_group_name=props.resource_group_name, + ) + + # Get connection string + connection_string = get_connection_string( + resource_group_name=props.resource_group_name, + storage_account=storage_account, + ) + + # Deploy service plan + app_service_plan = web.AppServicePlan( + f"{self._name}_app_service_plan", + kind="linux", + location=props.location, + name=f"{stack_name}-app-service-plan", + reserved=True, + resource_group_name=props.resource_group_name, + sku={ + "name": "B1", + "tier": "Basic", + "size": "B1", + "family": "B", + "capacity": 1, + }, + tags=child_tags, + ) + + # Deploy app + web.WebApp( + f"{self._name}_web_app", + enabled=True, + https_only=True, + kind="functionapp,linux", + location=props.location, + name=f"{stack_name}-gitea-mirror-api", + resource_group_name=props.resource_group_name, + server_farm_id=app_service_plan.id, + site_config=web.SiteConfigArgs( + always_on=True, + app_settings=[ + {"name": "AzureWebJobsStorage", "value": connection_string}, + {"name": "runtime", "value": "python"}, + {"name": "pythonVersion", "value": "3.11"}, + {"name": "FUNCTIONS_WORKER_RUNTIME", "value": "python"}, + {"name": "WEBSITE_RUN_FROM_PACKAGE", "value": blob_url}, + {"name": "FUNCTIONS_EXTENSION_VERSION", "value": "~4"}, + ], + linux_fx_version="Python|3.11", + ), + tags=child_tags, + ) + + +def get_blob_url( + blob: Input[storage.Blob], + container: Input[storage.BlobContainer], + resource_group_name: Input[str], + storage_account: Input[storage.StorageAccount], +) -> Output[str]: + sas = storage.list_storage_account_service_sas_output( + account_name=storage_account.name, + protocols=storage.HttpProtocol.HTTPS, + shared_access_expiry_time="2030-01-01", + shared_access_start_time="2021-01-01", + resource_group_name=resource_group_name, + # Access to container + resource=storage.SignedResource.C, + # Read access + permissions=storage.Permissions.R, + canonicalized_resource=Output.format( + "/blob/{account_name}/{container_name}", + account_name=storage_account.name, + container_name=container.name, + ), + # content_type="application/json", + # cache_control="max-age=5", + # content_disposition="inline", + # content_encoding="deflate", + ) + token = sas.service_sas_token + return Output.format( + "https://{storage_account_name}.blob.core.windows.net/{container_name}/{blob_name}?{token}", + storage_account_name=storage_account.name, + container_name=container.name, + blob_name=blob.name, + token=token, + ) + + +def get_connection_string( + resource_group_name: Input[str], + storage_account: Input[storage.StorageAccount], +) -> Output[str]: + storage_account_keys = storage.list_storage_account_keys_output( + resource_group_name=resource_group_name, + account_name=storage_account.name, + ) + primary_storage_key = storage_account_keys.keys[0].value + + return Output.format( + "DefaultEndpointsProtocol=https;AccountName={storage_account_name};AccountKey={primary_storage_key}", + storage_account_name=storage_account.name, + primary_storage_key=primary_storage_key, + ) diff --git a/data_safe_haven/resources/gitea_mirror/functions/.gitignore b/data_safe_haven/resources/gitea_mirror/functions/.gitignore new file mode 100644 index 0000000000..f15ac3fc66 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/.gitignore @@ -0,0 +1,48 @@ +bin +obj +csx +.vs +edge +Publish + +*.user +*.suo +*.cscfg +*.Cache +project.lock.json + +/packages +/TestResults + +/tools/NuGet.exe +/App_Data +/secrets +/data +.secrets +appsettings.json +local.settings.json + +node_modules +dist + +# Local python packages +.python_packages/ + +# Python Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Azurite artifacts +__blobstorage__ +__queuestorage__ +__azurite_db*__.json \ No newline at end of file diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py new file mode 100644 index 0000000000..4125e488e1 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py @@ -0,0 +1,87 @@ +import logging + +import azure.functions as func +import requests +from requests.auth import HTTPBasicAuth +from shared_code import ( + api_root, + check_args, + get_args, + gitea_host, + handle_response, + migrate_path, + missing_parameters_repsonse, + repos_path, + timeout, +) + + +def main(req: func.HttpRequest) -> func.HttpResponse: + logging.info("Request received.") + + raw_args = get_args( + [ + "address", + "name", + "password", + "username", + ], + req, + ) + args = check_args(raw_args) + if not args: + return missing_parameters_repsonse() + + extra_data = { + "description": f"Read-only mirror of {args['address']}", + "mirror": True, + "mirror_interval": "10m", + } + + auth = HTTPBasicAuth( + username=args["username"], + password=args["password"], + ) + + logging.info("Sending request to create mirror.") + + response = requests.post( + auth=auth, + data={ + "clone_addr": args["address"], + "repo_name": args["name"], + } + | extra_data, + timeout=timeout, + url=gitea_host + api_root + migrate_path, + ) + + if r := handle_response(response, [201], "Error creating repository."): + return r + + # Some arguments of the migrate endpoint seem to be ignored or overwritten. + # We set repository settings here. + logging.info("Sending request to configure mirror repo.") + + response = requests.patch( + auth=auth, + data={ + "has_actions": False, + "has_issues": False, + "has_packages": False, + "has_projects": False, + "has_pull_requests": False, + "has_releases": False, + "has_wiki": False, + }, + timeout=timeout, + url=gitea_host + api_root + repos_path + f"/{args['username']}/{args['name']}", + ) + + if r := handle_response(response, [200], "Error configuring repository."): + return r + + return func.HttpResponse( + "Mirror successfully created.", + status_code=200, + ) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function.json b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function.json new file mode 100644 index 0000000000..4667f0aca9 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py new file mode 100644 index 0000000000..a4aeb2f6b2 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py @@ -0,0 +1,52 @@ +import logging + +import azure.functions as func +import requests +from requests.auth import HTTPBasicAuth +from shared_code import ( + api_root, + check_args, + get_args, + gitea_host, + handle_response, + missing_parameters_repsonse, + repos_path, + timeout, +) + + +def main(req: func.HttpRequest) -> func.HttpResponse: + logging.info("Request received.") + + raw_args = get_args( + [ + "name", + "owner", + "password", + "username", + ], + req, + ) + args = check_args(raw_args) + if not args: + return missing_parameters_repsonse() + + auth = HTTPBasicAuth( + username=args["username"], + password=args["password"], + ) + + logging.info("Sending request to delete repository.") + response = requests.delete( + auth=auth, + timeout=timeout, + url=gitea_host + api_root + repos_path + f"/{args['owner']}/{args['name']}", + ) + + if r := handle_response(response, [204], "Error deleting repository."): + return r + + return func.HttpResponse( + "Repository successfully deleted.", + status_code=200, + ) diff --git a/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/function.json b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/function.json new file mode 100644 index 0000000000..4667f0aca9 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/data_safe_haven/resources/gitea_mirror/functions/host.json b/data_safe_haven/resources/gitea_mirror/functions/host.json new file mode 100644 index 0000000000..221cb15341 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/host.json @@ -0,0 +1,7 @@ +{ + "version": "2.0", + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} diff --git a/data_safe_haven/resources/gitea_mirror/functions/requirements.txt b/data_safe_haven/resources/gitea_mirror/functions/requirements.txt new file mode 100644 index 0000000000..f2293605cf --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/data_safe_haven/resources/gitea_mirror/functions/shared_code/__init__.py b/data_safe_haven/resources/gitea_mirror/functions/shared_code/__init__.py new file mode 100644 index 0000000000..557c93fce3 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/shared_code/__init__.py @@ -0,0 +1,60 @@ +import logging +from typing import Any + +import azure.functions as func +import requests + +app = func.FunctionApp() + +# Global parameters +# gitea_host = "http://gitea_mirror.local" +gitea_host = "http://localhost:3000" +api_root = "/api/v1" +migrate_path = "/repos/migrate" +repos_path = "/repos" +timeout = 60 + + +def get_args(args: list[str], req: func.HttpRequest) -> dict[str, str | None]: + try: + req_body = req.get_json() + except ValueError: + return {} + + args_dict = {arg: str_or_none(req_body.get(arg)) for arg in args} + logging.info(f"Parameters: {args}.") + return args_dict + + +def str_or_none(item: Any) -> str | None: + return str(item) if item is not None else None + + +def check_args(args: dict[str, str | None]) -> dict[str, str] | None: + if None in args.values(): + return None + else: + return {key: str(value) for key, value in args.items()} + + +def missing_parameters_repsonse() -> func.HttpResponse: + msg = "Required parameter not provided." + logging.critical(msg) + return func.HttpResponse( + msg, + status_code=400, + ) + + +def handle_response( + response: requests.Response, valid_codes: list[int], error_message: str +) -> func.HttpResponse | None: + logging.info(f"Response status code: {response.status_code}.") + logging.debug(f"Response contents: {response.text}.") + if response.status_code not in valid_codes: + return func.HttpResponse( + error_message, + status_code=400, + ) + else: + return None diff --git a/pyproject.toml b/pyproject.toml index 115fa5f42f..c22144279f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,6 @@ source = ["data_safe_haven/"] relative_files = true omit= [ "tests/*", - "data_safe_haven/resources/*", ] [tool.hatch.envs.default] @@ -91,6 +90,7 @@ detached = true dependencies = [ "ansible>=10.2.0", "ansible-dev-tools>=24.7.1", + "azure-functions>=1.20.0", "black>=24.1.0", "mypy>=1.0.0", "pydantic>=2.4", @@ -122,6 +122,7 @@ typing = "mypy {args:data_safe_haven}" [tool.hatch.envs.test] dependencies = [ + "azure-functions>=1.20.0", "coverage>=7.5.1", "freezegun>=1.5", "pytest>=8.1", diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py new file mode 100644 index 0000000000..10169703da --- /dev/null +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -0,0 +1,192 @@ +import json + +import azure.functions as func +from pytest import fixture + +from data_safe_haven.resources.gitea_mirror.functions.function_app import ( + api_root, + create_mirror, + delete_mirror, + gitea_host, + migrate_path, + repos_path, + str_or_none, +) + + +class TestStrOrNone: + def test_str_or_none(self): + assert str_or_none("hello") == "hello" + assert str_or_none(None) is None + + +@fixture +def create_mirror_func(): + return create_mirror.build().get_user_function() + + +@fixture +def delete_mirror_func(): + return delete_mirror.build().get_user_function() + + +class TestCreateMirror: + def test_create_mirror(self, create_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps( + { + "address": "https://github.com/user/repo", + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), + url="/api/create-mirror", + ) + + requests_mock.post(gitea_host + api_root + migrate_path, status_code=201) + requests_mock.patch( + gitea_host + api_root + repos_path + "/username/repo", status_code=200 + ) + + response = create_mirror_func(req) + assert response.status_code == 200 + assert b"Mirror successfully created." in response._HttpResponse__body + + def test_create_mirror_missing_args(self, create_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps( + { + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), + url="/api/create-mirror", + ) + + requests_mock.post(gitea_host + api_root + migrate_path, status_code=201) + requests_mock.patch( + gitea_host + api_root + repos_path + "/username/repo", status_code=200 + ) + + response = create_mirror_func(req) + assert response.status_code == 400 + assert b"Required parameter not provided." in response._HttpResponse__body + + def test_create_mirror_mirror_fail(self, create_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps( + { + "address": "https://github.com/user/repo", + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), + url="/api/create-mirror", + ) + + requests_mock.post(gitea_host + api_root + migrate_path, status_code=409) + requests_mock.patch( + gitea_host + api_root + repos_path + "/username/repo", status_code=200 + ) + + response = create_mirror_func(req) + assert response.status_code == 400 + assert b"Error creating repository." in response._HttpResponse__body + + def test_create_mirror_configure_fail(self, create_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps( + { + "address": "https://github.com/user/repo", + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), + url="/api/create-mirror", + ) + + requests_mock.post(gitea_host + api_root + migrate_path, status_code=201) + requests_mock.patch( + gitea_host + api_root + repos_path + "/username/repo", status_code=403 + ) + + response = create_mirror_func(req) + assert response.status_code == 400 + assert b"Error configuring repository." in response._HttpResponse__body + + +class TestDeleteMirror: + def test_delete_mirror(self, delete_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps( + { + "owner": "admin", + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), + url="/api/delete-mirror", + ) + + requests_mock.delete( + gitea_host + api_root + repos_path + "/admin/repo", + status_code=204, + ) + + response = delete_mirror_func(req) + assert response.status_code == 200 + + def test_delete_mirror_missing_args(self, delete_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps( + { + "name": "repo", + "owner": "admin", + "password": "password", + } + ).encode(), + url="/api/delete-mirror", + ) + + requests_mock.delete( + gitea_host + api_root + repos_path + "/admin/repo", + status_code=204, + ) + + response = delete_mirror_func(req) + assert response.status_code == 400 + assert b"Required parameter not provided." in response._HttpResponse__body + + def test_delete_mirror_fail(self, delete_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps( + { + "name": "repo", + "owner": "admin", + "password": "password", + "username": "admin", + } + ).encode(), + url="/api/delete-mirror", + ) + + requests_mock.delete( + gitea_host + api_root + repos_path + "/admin/repo", + status_code=404, + ) + + response = delete_mirror_func(req) + assert response.status_code == 400 + assert b"Error deleting repository." in response._HttpResponse__body diff --git a/typings/azure/functions/__init__.pyi b/typings/azure/functions/__init__.pyi new file mode 100644 index 0000000000..c96b5d80bc --- /dev/null +++ b/typings/azure/functions/__init__.pyi @@ -0,0 +1,13 @@ +from azure_functions import ( + AuthLevel, + FunctionApp, + HttpRequest, + HttpResponse, +) + +__all__ = [ + "AuthLevel", + "FunctionApp", + "HttpRequest", + "HttpResponse", +] diff --git a/typings/pulumi/__init__.pyi b/typings/pulumi/__init__.pyi index e1468220dd..b8e13de013 100644 --- a/typings/pulumi/__init__.pyi +++ b/typings/pulumi/__init__.pyi @@ -1,6 +1,6 @@ import pulumi.automation as automation import pulumi.dynamic as dynamic -from pulumi.asset import FileAsset +from pulumi.asset import FileArchive, FileAsset from pulumi.config import ( Config, ) @@ -22,6 +22,7 @@ __all__ = [ "Config", "dynamic", "export", + "FileArchive", "FileAsset", "Input", "Output", diff --git a/typings/pulumi_azure_native/__init__.pyi b/typings/pulumi_azure_native/__init__.pyi index 56be0a1e3a..bf031a21c7 100644 --- a/typings/pulumi_azure_native/__init__.pyi +++ b/typings/pulumi_azure_native/__init__.pyi @@ -15,6 +15,7 @@ import pulumi_azure_native.operationsmanagement as operationsmanagement import pulumi_azure_native.resources as resources import pulumi_azure_native.sql as sql import pulumi_azure_native.storage as storage +import pulumi_azure_native.web as web __all__ = [ "automation", @@ -33,5 +34,6 @@ __all__ = [ "resources", "sql", "storage", + "web", "_utilities", ]