From bc9c5ee7524ae513594c73fb76e1a5472d9b4af6 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 10:47:20 +0100 Subject: [PATCH 01/44] WIP: Add mirror creation script --- .../gitea_mirror/scripts/create_mirror.py | 49 +++++++++++++++++++ .../gitea_mirror/scripts/requirements.txt | 1 + 2 files changed, 50 insertions(+) create mode 100644 data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py create mode 100644 data_safe_haven/resources/gitea_mirror/scripts/requirements.txt diff --git a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py new file mode 100644 index 0000000000..66ed3bbfab --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py @@ -0,0 +1,49 @@ +import argparse + +import requests +from requests.auth import HTTPBasicAuth + +# gitea_host = "http://gitea_mirror.local" +gitea_host = "http://localhost:3000" +api_root = "/api/v1" +migrate_path = "/repos/migrate" +extra_data = { + "issues": False, + "mirror": False, + "mirror_interval": "600", + "pull_requests": False, + "releases": False, + "wiki": False, +} + +parser = argparse.ArgumentParser() +parser.add_argument( + 'username' +) +parser.add_argument( + 'password' +) +parser.add_argument( + 'name' +) +parser.add_argument( + 'address' +) +args = parser.parse_args() + +auth = HTTPBasicAuth( + username=args.username, + password=args.password, +) + +response = requests.post( + gitea_host + api_root + migrate_path, + auth=auth, + data={ + "clone_addr": args.address, + "repo_name": args.name, + } | extra_data, +) + +print(response.json()) +response.raise_for_status() diff --git a/data_safe_haven/resources/gitea_mirror/scripts/requirements.txt b/data_safe_haven/resources/gitea_mirror/scripts/requirements.txt new file mode 100644 index 0000000000..f2293605cf --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/scripts/requirements.txt @@ -0,0 +1 @@ +requests From c41196abbf523508de3325c71cadf47a67512f6b Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 11:06:53 +0100 Subject: [PATCH 02/44] Add description --- .../gitea_mirror/scripts/create_mirror.py | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py index 66ed3bbfab..2c3ea5ed04 100644 --- a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py +++ b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py @@ -3,19 +3,6 @@ import requests from requests.auth import HTTPBasicAuth -# gitea_host = "http://gitea_mirror.local" -gitea_host = "http://localhost:3000" -api_root = "/api/v1" -migrate_path = "/repos/migrate" -extra_data = { - "issues": False, - "mirror": False, - "mirror_interval": "600", - "pull_requests": False, - "releases": False, - "wiki": False, -} - parser = argparse.ArgumentParser() parser.add_argument( 'username' @@ -31,11 +18,29 @@ ) args = parser.parse_args() +# gitea_host = "http://gitea_mirror.local" +gitea_host = "http://localhost:3000" +api_root = "/api/v1" +migrate_path = "/repos/migrate" +extra_data = { + "description": f"Read-only mirror of {args.address}.", + "issues": False, + "mirror": False, + "mirror_interval": "0", + "pull_requests": False, + "releases": False, + "wiki": False, +} + auth = HTTPBasicAuth( username=args.username, password=args.password, ) +print( + {"clone_addr": args.address, "repo_name": args.name} | extra_data +) + response = requests.post( gitea_host + api_root + migrate_path, auth=auth, From d949996340a4243174da7a5ed748a994b8f6f49a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 11:07:09 +0100 Subject: [PATCH 03/44] Add delete script --- .../gitea_mirror/scripts/delete_repo.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py diff --git a/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py b/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py new file mode 100644 index 0000000000..6bb54e81f1 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py @@ -0,0 +1,39 @@ +import argparse + +import requests +from requests.auth import HTTPBasicAuth + +# gitea_host = "http://gitea_mirror.local" +gitea_host = "http://localhost:3000" +api_root = "/api/v1" +path = "/repos/" +extra_data = {} + +parser = argparse.ArgumentParser() +parser.add_argument( + 'username' +) +parser.add_argument( + 'password' +) +parser.add_argument( + 'owner' +) +parser.add_argument( + 'name' +) +args = parser.parse_args() + +auth = HTTPBasicAuth( + username=args.username, + password=args.password, +) + +response = requests.delete( + gitea_host + api_root + path + f"/{args.owner}/{args.name}", + auth=auth, + data={} | extra_data, +) + +# print(response.json()) +response.raise_for_status() From 9405f9a0653461aca3a2ed32528de1725f626e4f Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 11:25:01 +0100 Subject: [PATCH 04/44] Correct pull mirror creation --- .../resources/gitea_mirror/scripts/create_mirror.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py index 2c3ea5ed04..e9f145d00c 100644 --- a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py +++ b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py @@ -23,13 +23,9 @@ api_root = "/api/v1" migrate_path = "/repos/migrate" extra_data = { - "description": f"Read-only mirror of {args.address}.", - "issues": False, - "mirror": False, - "mirror_interval": "0", - "pull_requests": False, - "releases": False, - "wiki": False, + "description": f"Read-only mirror of {args.address}", + "mirror": True, + "mirror_interval": "10m", } auth = HTTPBasicAuth( @@ -42,12 +38,12 @@ ) response = requests.post( - gitea_host + api_root + migrate_path, auth=auth, data={ "clone_addr": args.address, "repo_name": args.name, } | extra_data, + url=gitea_host + api_root + migrate_path, ) print(response.json()) From 9a26635eaa300bdfa29b141cac1c017796b777d6 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 11:25:11 +0100 Subject: [PATCH 05/44] Configure pull mirror repository --- .../gitea_mirror/scripts/create_mirror.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py index e9f145d00c..4d18683ceb 100644 --- a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py +++ b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py @@ -22,6 +22,7 @@ gitea_host = "http://localhost:3000" api_root = "/api/v1" migrate_path = "/repos/migrate" +repos_path = "/repos" extra_data = { "description": f"Read-only mirror of {args.address}", "mirror": True, @@ -48,3 +49,21 @@ print(response.json()) response.raise_for_status() + +# Some arguments of the migrate endpoint seem to be ignored or overwritten +response = requests.patch( + auth=auth, + data={ + "has_actions": False, + "has_issues": False, + "has_packages": False, + "has_projects": False, + "has_pull_requests": False, + "has_releases": False, + "has_wiki": False, + }, + url=gitea_host + api_root + repos_path + f"/{args.username}/{args.name}", +) + +print(response.json()) +response.raise_for_status() From e8b2c17a9b4a85f75a8c644205fe137a4467617c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 11:30:57 +0100 Subject: [PATCH 06/44] Fix linting --- .../gitea_mirror/scripts/create_mirror.py | 30 +++++++------------ .../gitea_mirror/scripts/delete_repo.py | 21 +++++-------- 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py index 4d18683ceb..1b779bd49e 100644 --- a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py +++ b/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py @@ -4,18 +4,10 @@ from requests.auth import HTTPBasicAuth parser = argparse.ArgumentParser() -parser.add_argument( - 'username' -) -parser.add_argument( - 'password' -) -parser.add_argument( - 'name' -) -parser.add_argument( - 'address' -) +parser.add_argument("username") +parser.add_argument("password") +parser.add_argument("name") +parser.add_argument("address") args = parser.parse_args() # gitea_host = "http://gitea_mirror.local" @@ -28,26 +20,25 @@ "mirror": True, "mirror_interval": "10m", } +timeout = 60 auth = HTTPBasicAuth( username=args.username, password=args.password, ) -print( - {"clone_addr": args.address, "repo_name": args.name} | extra_data -) - response = requests.post( auth=auth, data={ "clone_addr": args.address, "repo_name": args.name, - } | extra_data, + } + | extra_data, + timeout=timeout, url=gitea_host + api_root + migrate_path, ) -print(response.json()) +print(response.json()) # noqa: T201 response.raise_for_status() # Some arguments of the migrate endpoint seem to be ignored or overwritten @@ -62,8 +53,9 @@ "has_releases": False, "has_wiki": False, }, + timeout=timeout, url=gitea_host + api_root + repos_path + f"/{args.username}/{args.name}", ) -print(response.json()) +print(response.json()) # noqa: T201 response.raise_for_status() diff --git a/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py b/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py index 6bb54e81f1..5a6e111635 100644 --- a/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py +++ b/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py @@ -8,20 +8,13 @@ api_root = "/api/v1" path = "/repos/" extra_data = {} +timeout = 60 parser = argparse.ArgumentParser() -parser.add_argument( - 'username' -) -parser.add_argument( - 'password' -) -parser.add_argument( - 'owner' -) -parser.add_argument( - 'name' -) +parser.add_argument("username") +parser.add_argument("password") +parser.add_argument("owner") +parser.add_argument("name") args = parser.parse_args() auth = HTTPBasicAuth( @@ -30,10 +23,10 @@ ) response = requests.delete( - gitea_host + api_root + path + f"/{args.owner}/{args.name}", auth=auth, data={} | extra_data, + timeout=timeout, + url=gitea_host + api_root + path + f"/{args.owner}/{args.name}", ) -# print(response.json()) response.raise_for_status() From 9e1a5f2e277474d9901677a9de39d4b5a008be5d Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 11:34:05 +0100 Subject: [PATCH 07/44] Remove unused variables --- data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py b/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py index 5a6e111635..511b09deb6 100644 --- a/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py +++ b/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py @@ -7,7 +7,6 @@ gitea_host = "http://localhost:3000" api_root = "/api/v1" path = "/repos/" -extra_data = {} timeout = 60 parser = argparse.ArgumentParser() @@ -24,7 +23,6 @@ response = requests.delete( auth=auth, - data={} | extra_data, timeout=timeout, url=gitea_host + api_root + path + f"/{args.owner}/{args.name}", ) From 7f8fe8769c8c74106788e54564f1f82a4c55f16b Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 15:08:35 +0100 Subject: [PATCH 08/44] Rewrite mirror creation as function app --- .../gitea_mirror/scripts/function_app.py | 107 ++++++++++++++++++ .../resources/gitea_mirror/scripts/host.json | 15 +++ .../gitea_mirror/scripts/requirements.txt | 5 + 3 files changed, 127 insertions(+) create mode 100644 data_safe_haven/resources/gitea_mirror/scripts/function_app.py create mode 100644 data_safe_haven/resources/gitea_mirror/scripts/host.json diff --git a/data_safe_haven/resources/gitea_mirror/scripts/function_app.py b/data_safe_haven/resources/gitea_mirror/scripts/function_app.py new file mode 100644 index 0000000000..12dbb516ad --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/scripts/function_app.py @@ -0,0 +1,107 @@ +import logging + +import azure.functions as func +import requests +from requests.auth import HTTPBasicAuth + +app = func.FunctionApp() + + +@app.route(route="HttpExample", auth_level=func.AuthLevel.ANONYMOUS) +def HttpExample(req: func.HttpRequest) -> func.HttpResponse: + logging.info("Request received.") + + address = req.params.get("address") + name = req.params.get('name') + password = req.params.get("password") + username = req.params.get("username") + logging.info(f"parameters: address={address}, name={name}, password={password}, username={username}") + + try: + req_body = req.get_json() + except ValueError: + pass + else: + address = req_body.get('address') + name = req_body.get('name') + password = req_body.get('password') + username = req_body.get('username') + logging.info(f"parameters: address={address}, name={name}, password={password}, username={username}") + + if (None in [address, name, password, username]): + msg = "Required parameter not provided." + logging.critical(msg) + return func.HttpResponse( + msg, + status_code=400, + ) + + # gitea_host = "http://gitea_mirror.local" + gitea_host = "http://localhost:3000" + api_root = "/api/v1" + migrate_path = "/repos/migrate" + repos_path = "/repos" + extra_data = { + "description": f"Read-only mirror of {address}", + "mirror": True, + "mirror_interval": "10m", + } + timeout = 60 + + auth = HTTPBasicAuth( + username=username, + password=password, + ) + + logging.info("Sending request to create mirror.") + + response = requests.post( + auth=auth, + data={ + "clone_addr": address, + "repo_name": name, + } + | extra_data, + timeout=timeout, + url=gitea_host + api_root + migrate_path, + ) + + logging.info(f"Response status code: {response.status_code}.") + logging.debug(f"Response contents: {response.json()}.") + if response.status_code != requests.codes.ok: + return func.HttpResponse( + "Error creating repository.", + status_code=400, + ) + + # Some arguments of the migrate endpoint seem to be ignored or overwritten. + # We set repository settings here. + logging.info("Sending request to configure mirror repo.") + + response = requests.patch( + auth=auth, + data={ + "has_actions": False, + "has_issues": False, + "has_packages": False, + "has_projects": False, + "has_pull_requests": False, + "has_releases": False, + "has_wiki": False, + }, + timeout=timeout, + url=gitea_host + api_root + repos_path + f"/{username}/{name}", + ) + + logging.info(f"Response status code: {response.status_code}.") + logging.debug(f"Response contents: {response.json()}.") + if response.status_code != requests.codes.ok: + return func.HttpResponse( + "Error configuring repository.", + status_code=400, + ) + + return func.HttpResponse( + "Mirror successfully created", + status_code=200, + ) diff --git a/data_safe_haven/resources/gitea_mirror/scripts/host.json b/data_safe_haven/resources/gitea_mirror/scripts/host.json new file mode 100644 index 0000000000..06d01bdaa9 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/scripts/host.json @@ -0,0 +1,15 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} diff --git a/data_safe_haven/resources/gitea_mirror/scripts/requirements.txt b/data_safe_haven/resources/gitea_mirror/scripts/requirements.txt index f2293605cf..b1f67106a3 100644 --- a/data_safe_haven/resources/gitea_mirror/scripts/requirements.txt +++ b/data_safe_haven/resources/gitea_mirror/scripts/requirements.txt @@ -1 +1,6 @@ +# Do not include azure-functions-worker in this file +# The Python Worker is managed by the Azure Functions platform +# Manually managing azure-functions-worker may cause unexpected issues + +azure-functions requests From fabd2b26a2c75cbe872bbd588a246d35f269ef7f Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 15:11:36 +0100 Subject: [PATCH 09/44] Restructure directory --- .../{scripts => functions}/create_mirror.py | 0 .../functions/create_mirror/.gitignore | 48 +++++++++++++++++++ .../create_mirror}/function_app.py | 0 .../create_mirror}/host.json | 0 .../create_mirror}/requirements.txt | 0 .../delete_repo}/delete_repo.py | 0 6 files changed, 48 insertions(+) rename data_safe_haven/resources/gitea_mirror/{scripts => functions}/create_mirror.py (100%) create mode 100644 data_safe_haven/resources/gitea_mirror/functions/create_mirror/.gitignore rename data_safe_haven/resources/gitea_mirror/{scripts => functions/create_mirror}/function_app.py (100%) rename data_safe_haven/resources/gitea_mirror/{scripts => functions/create_mirror}/host.json (100%) rename data_safe_haven/resources/gitea_mirror/{scripts => functions/create_mirror}/requirements.txt (100%) rename data_safe_haven/resources/gitea_mirror/{scripts => functions/delete_repo}/delete_repo.py (100%) diff --git a/data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror.py similarity index 100% rename from data_safe_haven/resources/gitea_mirror/scripts/create_mirror.py rename to data_safe_haven/resources/gitea_mirror/functions/create_mirror.py diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/.gitignore b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/.gitignore new file mode 100644 index 0000000000..f15ac3fc66 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/.gitignore @@ -0,0 +1,48 @@ +bin +obj +csx +.vs +edge +Publish + +*.user +*.suo +*.cscfg +*.Cache +project.lock.json + +/packages +/TestResults + +/tools/NuGet.exe +/App_Data +/secrets +/data +.secrets +appsettings.json +local.settings.json + +node_modules +dist + +# Local python packages +.python_packages/ + +# Python Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Azurite artifacts +__blobstorage__ +__queuestorage__ +__azurite_db*__.json \ No newline at end of file diff --git a/data_safe_haven/resources/gitea_mirror/scripts/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py similarity index 100% rename from data_safe_haven/resources/gitea_mirror/scripts/function_app.py rename to data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py diff --git a/data_safe_haven/resources/gitea_mirror/scripts/host.json b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/host.json similarity index 100% rename from data_safe_haven/resources/gitea_mirror/scripts/host.json rename to data_safe_haven/resources/gitea_mirror/functions/create_mirror/host.json diff --git a/data_safe_haven/resources/gitea_mirror/scripts/requirements.txt b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/requirements.txt similarity index 100% rename from data_safe_haven/resources/gitea_mirror/scripts/requirements.txt rename to data_safe_haven/resources/gitea_mirror/functions/create_mirror/requirements.txt diff --git a/data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py b/data_safe_haven/resources/gitea_mirror/functions/delete_repo/delete_repo.py similarity index 100% rename from data_safe_haven/resources/gitea_mirror/scripts/delete_repo.py rename to data_safe_haven/resources/gitea_mirror/functions/delete_repo/delete_repo.py From 68e85f236a876326260f828990e606442aa52cda Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 15:15:57 +0100 Subject: [PATCH 10/44] Rename function and api root --- .../functions/create_mirror/function_app.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py index 12dbb516ad..245ae2e93b 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py @@ -7,28 +7,32 @@ app = func.FunctionApp() -@app.route(route="HttpExample", auth_level=func.AuthLevel.ANONYMOUS) -def HttpExample(req: func.HttpRequest) -> func.HttpResponse: +@app.route(route="create-mirror", auth_level=func.AuthLevel.ANONYMOUS) +def create_mirror(req: func.HttpRequest) -> func.HttpResponse: # logging.info("Request received.") address = req.params.get("address") - name = req.params.get('name') + name = req.params.get("name") password = req.params.get("password") username = req.params.get("username") - logging.info(f"parameters: address={address}, name={name}, password={password}, username={username}") + logging.info( + f"parameters: address={address}, name={name}, password={password}, username={username}" + ) try: req_body = req.get_json() except ValueError: pass else: - address = req_body.get('address') - name = req_body.get('name') - password = req_body.get('password') - username = req_body.get('username') - logging.info(f"parameters: address={address}, name={name}, password={password}, username={username}") + address = req_body.get("address") + name = req_body.get("name") + password = req_body.get("password") + username = req_body.get("username") + logging.info( + f"parameters: address={address}, name={name}, password={password}, username={username}" + ) - if (None in [address, name, password, username]): + if None in [address, name, password, username]: msg = "Required parameter not provided." logging.critical(msg) return func.HttpResponse( From 3b15ff221d5a48e5bcba8f198974a0b56c2a4fdf Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 15:25:26 +0100 Subject: [PATCH 11/44] Remove unused method to get args --- .../gitea_mirror/functions/create_mirror/function_app.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py index 245ae2e93b..56ab17335b 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py @@ -11,14 +11,6 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: # logging.info("Request received.") - address = req.params.get("address") - name = req.params.get("name") - password = req.params.get("password") - username = req.params.get("username") - logging.info( - f"parameters: address={address}, name={name}, password={password}, username={username}" - ) - try: req_body = req.get_json() except ValueError: From 783999e1c6ea4693148c4c37e5c6b28f2146da15 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 15:25:41 +0100 Subject: [PATCH 12/44] Correct status code check --- .../gitea_mirror/functions/create_mirror/function_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py index 56ab17335b..81322fd91a 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py @@ -64,7 +64,7 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: # logging.info(f"Response status code: {response.status_code}.") logging.debug(f"Response contents: {response.json()}.") - if response.status_code != requests.codes.ok: + if response.status_code != 201: return func.HttpResponse( "Error creating repository.", status_code=400, From 5ad78a45e20c09702b84614341870a421c38a75a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 15:30:18 +0100 Subject: [PATCH 13/44] Add exception for magic number --- .../gitea_mirror/functions/create_mirror/function_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py index 81322fd91a..714389d672 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py @@ -64,7 +64,7 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: # logging.info(f"Response status code: {response.status_code}.") logging.debug(f"Response contents: {response.json()}.") - if response.status_code != 201: + if response.status_code != 201: # noqa: PLR2004 return func.HttpResponse( "Error creating repository.", status_code=400, From bef8bccc448c04ca4e2f9a8be9348e04ab7077b7 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 15:39:45 +0100 Subject: [PATCH 14/44] Add typings for azure functions --- pyproject.toml | 1 + typings/azure/functions/__init__.pyi | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 typings/azure/functions/__init__.pyi diff --git a/pyproject.toml b/pyproject.toml index 4f302a7719..8a3e2b7de4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ build = [ [tool.hatch.envs.lint] detached = true dependencies = [ + "azure-functions>=1.20.0", "black>=24.1.0", "mypy>=1.0.0", "pydantic>=2.4", diff --git a/typings/azure/functions/__init__.pyi b/typings/azure/functions/__init__.pyi new file mode 100644 index 0000000000..c96b5d80bc --- /dev/null +++ b/typings/azure/functions/__init__.pyi @@ -0,0 +1,13 @@ +from azure_functions import ( + AuthLevel, + FunctionApp, + HttpRequest, + HttpResponse, +) + +__all__ = [ + "AuthLevel", + "FunctionApp", + "HttpRequest", + "HttpResponse", +] From 00f30365af188bc30d0a63f8b95d487376f5424c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 10 Jul 2024 15:43:13 +0100 Subject: [PATCH 15/44] Remove empty comment --- .../gitea_mirror/functions/create_mirror/function_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py index 714389d672..83217feb72 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py @@ -8,7 +8,7 @@ @app.route(route="create-mirror", auth_level=func.AuthLevel.ANONYMOUS) -def create_mirror(req: func.HttpRequest) -> func.HttpResponse: # +def create_mirror(req: func.HttpRequest) -> func.HttpResponse: logging.info("Request received.") try: From f453d39f99f59b2e7c9b437882cd64c44b1762cb Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 10:26:07 +0100 Subject: [PATCH 16/44] Remove old script --- .../gitea_mirror/functions/create_mirror.py | 61 ------------------- 1 file changed, 61 deletions(-) delete mode 100644 data_safe_haven/resources/gitea_mirror/functions/create_mirror.py diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror.py deleted file mode 100644 index 1b779bd49e..0000000000 --- a/data_safe_haven/resources/gitea_mirror/functions/create_mirror.py +++ /dev/null @@ -1,61 +0,0 @@ -import argparse - -import requests -from requests.auth import HTTPBasicAuth - -parser = argparse.ArgumentParser() -parser.add_argument("username") -parser.add_argument("password") -parser.add_argument("name") -parser.add_argument("address") -args = parser.parse_args() - -# gitea_host = "http://gitea_mirror.local" -gitea_host = "http://localhost:3000" -api_root = "/api/v1" -migrate_path = "/repos/migrate" -repos_path = "/repos" -extra_data = { - "description": f"Read-only mirror of {args.address}", - "mirror": True, - "mirror_interval": "10m", -} -timeout = 60 - -auth = HTTPBasicAuth( - username=args.username, - password=args.password, -) - -response = requests.post( - auth=auth, - data={ - "clone_addr": args.address, - "repo_name": args.name, - } - | extra_data, - timeout=timeout, - url=gitea_host + api_root + migrate_path, -) - -print(response.json()) # noqa: T201 -response.raise_for_status() - -# Some arguments of the migrate endpoint seem to be ignored or overwritten -response = requests.patch( - auth=auth, - data={ - "has_actions": False, - "has_issues": False, - "has_packages": False, - "has_projects": False, - "has_pull_requests": False, - "has_releases": False, - "has_wiki": False, - }, - timeout=timeout, - url=gitea_host + api_root + repos_path + f"/{args.username}/{args.name}", -) - -print(response.json()) # noqa: T201 -response.raise_for_status() From 428c649653f8e28127971eca614dd5d621b2d899 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 10:44:43 +0100 Subject: [PATCH 17/44] Put both routes in one function app --- .../functions/{create_mirror => }/.gitignore | 0 .../functions/delete_repo/delete_repo.py | 30 ---------- .../{create_mirror => }/function_app.py | 56 +++++++++++++++++-- .../functions/{create_mirror => }/host.json | 0 .../{create_mirror => }/requirements.txt | 0 5 files changed, 50 insertions(+), 36 deletions(-) rename data_safe_haven/resources/gitea_mirror/functions/{create_mirror => }/.gitignore (100%) delete mode 100644 data_safe_haven/resources/gitea_mirror/functions/delete_repo/delete_repo.py rename data_safe_haven/resources/gitea_mirror/functions/{create_mirror => }/function_app.py (65%) rename data_safe_haven/resources/gitea_mirror/functions/{create_mirror => }/host.json (100%) rename data_safe_haven/resources/gitea_mirror/functions/{create_mirror => }/requirements.txt (100%) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/.gitignore b/data_safe_haven/resources/gitea_mirror/functions/.gitignore similarity index 100% rename from data_safe_haven/resources/gitea_mirror/functions/create_mirror/.gitignore rename to data_safe_haven/resources/gitea_mirror/functions/.gitignore diff --git a/data_safe_haven/resources/gitea_mirror/functions/delete_repo/delete_repo.py b/data_safe_haven/resources/gitea_mirror/functions/delete_repo/delete_repo.py deleted file mode 100644 index 511b09deb6..0000000000 --- a/data_safe_haven/resources/gitea_mirror/functions/delete_repo/delete_repo.py +++ /dev/null @@ -1,30 +0,0 @@ -import argparse - -import requests -from requests.auth import HTTPBasicAuth - -# gitea_host = "http://gitea_mirror.local" -gitea_host = "http://localhost:3000" -api_root = "/api/v1" -path = "/repos/" -timeout = 60 - -parser = argparse.ArgumentParser() -parser.add_argument("username") -parser.add_argument("password") -parser.add_argument("owner") -parser.add_argument("name") -args = parser.parse_args() - -auth = HTTPBasicAuth( - username=args.username, - password=args.password, -) - -response = requests.delete( - auth=auth, - timeout=timeout, - url=gitea_host + api_root + path + f"/{args.owner}/{args.name}", -) - -response.raise_for_status() diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/function_app.py similarity index 65% rename from data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py rename to data_safe_haven/resources/gitea_mirror/functions/function_app.py index 83217feb72..f1c860d08d 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/function_app.py @@ -6,6 +6,13 @@ app = func.FunctionApp() +# gitea_host = "http://gitea_mirror.local" +gitea_host = "http://localhost:3000" +api_root = "/api/v1" +migrate_path = "/repos/migrate" +repos_path = "/repos" +timeout = 60 + @app.route(route="create-mirror", auth_level=func.AuthLevel.ANONYMOUS) def create_mirror(req: func.HttpRequest) -> func.HttpResponse: @@ -32,17 +39,11 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: status_code=400, ) - # gitea_host = "http://gitea_mirror.local" - gitea_host = "http://localhost:3000" - api_root = "/api/v1" - migrate_path = "/repos/migrate" - repos_path = "/repos" extra_data = { "description": f"Read-only mirror of {address}", "mirror": True, "mirror_interval": "10m", } - timeout = 60 auth = HTTPBasicAuth( username=username, @@ -101,3 +102,46 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: "Mirror successfully created", status_code=200, ) + + +@app.route(route="delete-mirror", auth_level=func.AuthLevel.ANONYMOUS) +def delete_mirror(req: func.HttpRequest) -> func.HttpResponse: + logging.info("Request received.") + + try: + req_body = req.get_json() + except ValueError: + pass + else: + name = req_body.get("name") + owner = req_body.get("owner") + password = req_body.get("password") + username = req_body.get("username") + logging.info( + f"parameters: name={name}, owner={owner}, password={password}, username={username}" + ) + + auth = HTTPBasicAuth( + username=username, + password=password, + ) + + logging.info("Sending request to delete repository.") + response = requests.delete( + auth=auth, + timeout=timeout, + url=gitea_host + api_root + repos_path + f"/{owner}/{name}", + ) + + logging.info(f"Response status code: {response.status_code}.") + logging.debug(f"Response contents: {response.content}.") + if response.status_code != 204: # noqa: PLR2004 + return func.HttpResponse( + "Error configuring repository.", + status_code=400, + ) + + return func.HttpResponse( + "Repository successfully deleted.", + status_code=200, + ) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/host.json b/data_safe_haven/resources/gitea_mirror/functions/host.json similarity index 100% rename from data_safe_haven/resources/gitea_mirror/functions/create_mirror/host.json rename to data_safe_haven/resources/gitea_mirror/functions/host.json diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/requirements.txt b/data_safe_haven/resources/gitea_mirror/functions/requirements.txt similarity index 100% rename from data_safe_haven/resources/gitea_mirror/functions/create_mirror/requirements.txt rename to data_safe_haven/resources/gitea_mirror/functions/requirements.txt From 2e2dae333164072f4e7d8fec9c371fcc915de773 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 11:26:06 +0100 Subject: [PATCH 18/44] Generalise argument parsing --- .../gitea_mirror/functions/function_app.py | 105 +++++++++++------- 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/function_app.py index f1c860d08d..43470201c7 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/function_app.py @@ -1,4 +1,5 @@ import logging +from typing import Any import azure.functions as func import requests @@ -6,6 +7,7 @@ app = func.FunctionApp() +# Global parameters # gitea_host = "http://gitea_mirror.local" gitea_host = "http://localhost:3000" api_root = "/api/v1" @@ -14,40 +16,63 @@ timeout = 60 -@app.route(route="create-mirror", auth_level=func.AuthLevel.ANONYMOUS) -def create_mirror(req: func.HttpRequest) -> func.HttpResponse: - logging.info("Request received.") - +def get_args(args: list[str], req: func.HttpRequest) -> dict[str, str | None]: try: req_body = req.get_json() except ValueError: - pass + return {} + + args_dict = {arg: str_or_none(req_body.get(arg)) for arg in args} + logging.info(f"Parameters: {args}.") + return args_dict + + +def str_or_none(item: Any) -> str | None: + return str(item) if item is not None else None + + +def check_args(args: dict[str, str | None]) -> dict[str, str] | None: + if None in args.values(): + return None else: - address = req_body.get("address") - name = req_body.get("name") - password = req_body.get("password") - username = req_body.get("username") - logging.info( - f"parameters: address={address}, name={name}, password={password}, username={username}" + return {key: str(value) for key, value in args.items()} + + +def missing_parameters_repsonse() -> func.HttpResponse: + msg = "Required parameter not provided." + logging.critical(msg) + return func.HttpResponse( + msg, + status_code=400, ) - if None in [address, name, password, username]: - msg = "Required parameter not provided." - logging.critical(msg) - return func.HttpResponse( - msg, - status_code=400, - ) + +@app.route(route="create-mirror", auth_level=func.AuthLevel.ANONYMOUS) +def create_mirror(req: func.HttpRequest) -> func.HttpResponse: + logging.info("Request received.") + + raw_args = get_args( + [ + "address", + "name", + "password", + "username", + ], + req, + ) + args = check_args(raw_args) + if not args: + return missing_parameters_repsonse() extra_data = { - "description": f"Read-only mirror of {address}", + "description": f"Read-only mirror of {args['address']}", "mirror": True, "mirror_interval": "10m", } auth = HTTPBasicAuth( - username=username, - password=password, + username=args["username"], + password=args["password"], ) logging.info("Sending request to create mirror.") @@ -55,8 +80,8 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: response = requests.post( auth=auth, data={ - "clone_addr": address, - "repo_name": name, + "clone_addr": args["address"], + "repo_name": args["name"], } | extra_data, timeout=timeout, @@ -87,7 +112,7 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: "has_wiki": False, }, timeout=timeout, - url=gitea_host + api_root + repos_path + f"/{username}/{name}", + url=gitea_host + api_root + repos_path + f"/{args['username']}/{args['name']}", ) logging.info(f"Response status code: {response.status_code}.") @@ -108,36 +133,36 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: def delete_mirror(req: func.HttpRequest) -> func.HttpResponse: logging.info("Request received.") - try: - req_body = req.get_json() - except ValueError: - pass - else: - name = req_body.get("name") - owner = req_body.get("owner") - password = req_body.get("password") - username = req_body.get("username") - logging.info( - f"parameters: name={name}, owner={owner}, password={password}, username={username}" + raw_args = get_args( + [ + "name", + "owner", + "password", + "username", + ], + req, ) + args = check_args(raw_args) + if not args: + return missing_parameters_repsonse() auth = HTTPBasicAuth( - username=username, - password=password, + username=args["username"], + password=args["password"], ) logging.info("Sending request to delete repository.") response = requests.delete( auth=auth, timeout=timeout, - url=gitea_host + api_root + repos_path + f"/{owner}/{name}", + url=gitea_host + api_root + repos_path + f"/{args['owner']}/{args['name']}", ) logging.info(f"Response status code: {response.status_code}.") - logging.debug(f"Response contents: {response.content}.") + logging.debug(f"Response contents: {response.text}.") if response.status_code != 204: # noqa: PLR2004 return func.HttpResponse( - "Error configuring repository.", + "Error deleting repository.", status_code=400, ) From eb7bdab6659e319ec45c3d4941ffc7fc90610ab5 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 11:34:01 +0100 Subject: [PATCH 19/44] Generalise response handling --- .../gitea_mirror/functions/function_app.py | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/function_app.py index 43470201c7..b2a503b590 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/function_app.py @@ -47,6 +47,18 @@ def missing_parameters_repsonse() -> func.HttpResponse: ) +def handle_response( + response: requests.Response, valid_codes: list[int], error_message: str +) -> func.HttpResponse | None: + logging.info(f"Response status code: {response.status_code}.") + logging.debug(f"Response contents: {response.text}.") + if response.status_code not in valid_codes: + return func.HttpResponse( + error_message, + status_code=400, + ) + + @app.route(route="create-mirror", auth_level=func.AuthLevel.ANONYMOUS) def create_mirror(req: func.HttpRequest) -> func.HttpResponse: logging.info("Request received.") @@ -88,13 +100,8 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: url=gitea_host + api_root + migrate_path, ) - logging.info(f"Response status code: {response.status_code}.") - logging.debug(f"Response contents: {response.json()}.") - if response.status_code != 201: # noqa: PLR2004 - return func.HttpResponse( - "Error creating repository.", - status_code=400, - ) + if r := handle_response(response, [201], "Error creating repository."): + return r # Some arguments of the migrate endpoint seem to be ignored or overwritten. # We set repository settings here. @@ -115,13 +122,8 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: url=gitea_host + api_root + repos_path + f"/{args['username']}/{args['name']}", ) - logging.info(f"Response status code: {response.status_code}.") - logging.debug(f"Response contents: {response.json()}.") - if response.status_code != requests.codes.ok: - return func.HttpResponse( - "Error configuring repository.", - status_code=400, - ) + if r := handle_response(response, [200], "Error configuring repository."): + return r return func.HttpResponse( "Mirror successfully created", @@ -158,13 +160,8 @@ def delete_mirror(req: func.HttpRequest) -> func.HttpResponse: url=gitea_host + api_root + repos_path + f"/{args['owner']}/{args['name']}", ) - logging.info(f"Response status code: {response.status_code}.") - logging.debug(f"Response contents: {response.text}.") - if response.status_code != 204: # noqa: PLR2004 - return func.HttpResponse( - "Error deleting repository.", - status_code=400, - ) + if r := handle_response(response, [204], "Error deleting repository."): + return r return func.HttpResponse( "Repository successfully deleted.", From 99dc8cb8fb69665a3a39be1385ebe5516780b063 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 11:45:29 +0100 Subject: [PATCH 20/44] Add return statement --- .../resources/gitea_mirror/functions/function_app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_safe_haven/resources/gitea_mirror/functions/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/function_app.py index b2a503b590..54bdc33e09 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/function_app.py @@ -57,6 +57,8 @@ def handle_response( error_message, status_code=400, ) + else: + return None @app.route(route="create-mirror", auth_level=func.AuthLevel.ANONYMOUS) From 0a415553135c42c57d0cfbedf3f0b10ae22086cc Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 12:01:21 +0100 Subject: [PATCH 21/44] Add basic test --- .../gitea_mirror/functions/test_function_app.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/resources/gitea_mirror/functions/test_function_app.py diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py new file mode 100644 index 0000000000..7c23ad5f9a --- /dev/null +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -0,0 +1,11 @@ +# import azure.functions as func + +from data_safe_haven.resources.gitea_mirror.functions.function_app import ( + str_or_none +) + + +class TestStrOrNone: + def test_str_or_none(self): + assert str_or_none("hello") == "hello" + assert str_or_none(None) is None From fb4127827fec9077031ee671c588f2e77c5ba88b Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 15:17:10 +0100 Subject: [PATCH 22/44] Add delete mirror test --- pyproject.toml | 1 + .../functions/test_function_app.py | 31 +++++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8a3e2b7de4..7aae01e041 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,7 @@ all = [ [tool.hatch.envs.test] dependencies = [ + "azure-functions>=1.20.0", "coverage>=7.5.1", "freezegun>=1.5", "pytest>=8.1", diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py index 7c23ad5f9a..24ab8b5670 100644 --- a/tests/resources/gitea_mirror/functions/test_function_app.py +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -1,7 +1,10 @@ -# import azure.functions as func +import json + +import azure.functions as func from data_safe_haven.resources.gitea_mirror.functions.function_app import ( - str_or_none + str_or_none, + delete_mirror, ) @@ -9,3 +12,27 @@ class TestStrOrNone: def test_str_or_none(self): assert str_or_none("hello") == "hello" assert str_or_none(None) is None + + +class TestDeleteMirror: + def test_delete_mirror(self, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps({ + "name": "repo", + "owner": "admin", + "password": "password", + "username": "username", + }).encode(), + url="/api/delete-mirror", + ) + + function = delete_mirror.build().get_user_function() + + requests_mock.delete( + "http://localhost:3000/api/v1/repos/admin/repo", + status_code=204, + ) + + response = function(req) + assert response.status_code == 200 From 003d24feccd761c5b6426d92ad0a20e61a5b3f92 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 15:26:44 +0100 Subject: [PATCH 23/44] Add test for missing arguments --- .../functions/test_function_app.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py index 24ab8b5670..7c265ecabf 100644 --- a/tests/resources/gitea_mirror/functions/test_function_app.py +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -36,3 +36,25 @@ def test_delete_mirror(self, requests_mock): response = function(req) assert response.status_code == 200 + + def test_delete_mirror_missing_args(self, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps({ + "name": "repo", + "owner": "admin", + "password": "password", + }).encode(), + url="/api/delete-mirror", + ) + + function = delete_mirror.build().get_user_function() + + requests_mock.delete( + "http://localhost:3000/api/v1/repos/admin/repo", + status_code=204, + ) + + response = function(req) + assert response.status_code == 400 + assert b"Required parameter not provided." in response._HttpResponse__body From 8b63b8ead715a4d285a0ffc296d37a7e9ba2466a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 15:42:50 +0100 Subject: [PATCH 24/44] Add basic test for create mirror --- .../gitea_mirror/functions/function_app.py | 2 +- .../functions/test_function_app.py | 53 +++++++++++++++---- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/function_app.py index 54bdc33e09..28410913d6 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/function_app.py +++ b/data_safe_haven/resources/gitea_mirror/functions/function_app.py @@ -128,7 +128,7 @@ def create_mirror(req: func.HttpRequest) -> func.HttpResponse: return r return func.HttpResponse( - "Mirror successfully created", + "Mirror successfully created.", status_code=200, ) diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py index 7c265ecabf..dd0b5d588e 100644 --- a/tests/resources/gitea_mirror/functions/test_function_app.py +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -1,9 +1,11 @@ import json import azure.functions as func +from pytest import fixture from data_safe_haven.resources.gitea_mirror.functions.function_app import ( str_or_none, + create_mirror, delete_mirror, ) @@ -14,30 +16,65 @@ def test_str_or_none(self): assert str_or_none(None) is None -class TestDeleteMirror: - def test_delete_mirror(self, requests_mock): +@fixture +def create_mirror_func(): + return create_mirror.build().get_user_function() + + +@fixture +def delete_mirror_func(): + return delete_mirror.build().get_user_function() + + +class TestCreateMirror: + def test_create_mirror(self, create_mirror_func, requests_mock): req = func.HttpRequest( method="POST", body=json.dumps({ + "address": "https://github.com/user/repo", "name": "repo", - "owner": "admin", "password": "password", "username": "username", }).encode(), url="/api/delete-mirror", ) - function = delete_mirror.build().get_user_function() + requests_mock.post( + "http://localhost:3000/api/v1/repos/migrate", + status_code=201 + ) + requests_mock.patch( + "http://localhost:3000/api/v1/repos/username/repo", + status_code=200 + ) + + response = create_mirror_func(req) + assert response.status_code == 200 + assert b"Mirror successfully created." in response._HttpResponse__body + + +class TestDeleteMirror: + def test_delete_mirror(self, delete_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps({ + "owner": "admin", + "name": "repo", + "password": "password", + "username": "username", + }).encode(), + url="/api/delete-mirror", + ) requests_mock.delete( "http://localhost:3000/api/v1/repos/admin/repo", status_code=204, ) - response = function(req) + response = delete_mirror_func(req) assert response.status_code == 200 - def test_delete_mirror_missing_args(self, requests_mock): + def test_delete_mirror_missing_args(self, delete_mirror_func, requests_mock): req = func.HttpRequest( method="POST", body=json.dumps({ @@ -48,13 +85,11 @@ def test_delete_mirror_missing_args(self, requests_mock): url="/api/delete-mirror", ) - function = delete_mirror.build().get_user_function() - requests_mock.delete( "http://localhost:3000/api/v1/repos/admin/repo", status_code=204, ) - response = function(req) + response = delete_mirror_func(req) assert response.status_code == 400 assert b"Required parameter not provided." in response._HttpResponse__body From b88a38094a0d2b2fa68d05d7912f21e72caa9a08 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 15:47:33 +0100 Subject: [PATCH 25/44] Add test for failure to delete mirror --- .../functions/test_function_app.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py index dd0b5d588e..c41600ffdf 100644 --- a/tests/resources/gitea_mirror/functions/test_function_app.py +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -93,3 +93,24 @@ def test_delete_mirror_missing_args(self, delete_mirror_func, requests_mock): response = delete_mirror_func(req) assert response.status_code == 400 assert b"Required parameter not provided." in response._HttpResponse__body + + def test_delete_mirror_fail(self, delete_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps({ + "name": "repo", + "owner": "admin", + "password": "password", + "username": "admin", + }).encode(), + url="/api/delete-mirror", + ) + + requests_mock.delete( + "http://localhost:3000/api/v1/repos/admin/repo", + status_code=404, + ) + + response = delete_mirror_func(req) + assert response.status_code == 400 + assert b"Error deleting repository." in response._HttpResponse__body From 76e5f4497b3bb65c791be92a7edbc323cad48962 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 15:52:16 +0100 Subject: [PATCH 26/44] Add failure tests for mirror creation --- .../functions/test_function_app.py | 76 ++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py index c41600ffdf..07617509af 100644 --- a/tests/resources/gitea_mirror/functions/test_function_app.py +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -36,7 +36,7 @@ def test_create_mirror(self, create_mirror_func, requests_mock): "password": "password", "username": "username", }).encode(), - url="/api/delete-mirror", + url="/api/create-mirror", ) requests_mock.post( @@ -52,6 +52,80 @@ def test_create_mirror(self, create_mirror_func, requests_mock): assert response.status_code == 200 assert b"Mirror successfully created." in response._HttpResponse__body + def test_create_mirror_missing_args(self, create_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps({ + "name": "repo", + "password": "password", + "username": "username", + }).encode(), + url="/api/create-mirror", + ) + + requests_mock.post( + "http://localhost:3000/api/v1/repos/migrate", + status_code=201 + ) + requests_mock.patch( + "http://localhost:3000/api/v1/repos/username/repo", + status_code=200 + ) + + response = create_mirror_func(req) + assert response.status_code == 400 + assert b"Required parameter not provided." in response._HttpResponse__body + + def test_create_mirror_mirror_fail(self, create_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps({ + "address": "https://github.com/user/repo", + "name": "repo", + "password": "password", + "username": "username", + }).encode(), + url="/api/create-mirror", + ) + + requests_mock.post( + "http://localhost:3000/api/v1/repos/migrate", + status_code=409 + ) + requests_mock.patch( + "http://localhost:3000/api/v1/repos/username/repo", + status_code=200 + ) + + response = create_mirror_func(req) + assert response.status_code == 400 + assert b"Error creating repository." in response._HttpResponse__body + + def test_create_mirror_configure_fail(self, create_mirror_func, requests_mock): + req = func.HttpRequest( + method="POST", + body=json.dumps({ + "address": "https://github.com/user/repo", + "name": "repo", + "password": "password", + "username": "username", + }).encode(), + url="/api/create-mirror", + ) + + requests_mock.post( + "http://localhost:3000/api/v1/repos/migrate", + status_code=201 + ) + requests_mock.patch( + "http://localhost:3000/api/v1/repos/username/repo", + status_code=403 + ) + + response = create_mirror_func(req) + assert response.status_code == 400 + assert b"Error configuring repository." in response._HttpResponse__body + class TestDeleteMirror: def test_delete_mirror(self, delete_mirror_func, requests_mock): From edd65dcca5a838bba53799f9ba3ebb761304250b Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 15:56:29 +0100 Subject: [PATCH 27/44] Remove some magic strings --- .../functions/test_function_app.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py index 07617509af..3eb8038fd4 100644 --- a/tests/resources/gitea_mirror/functions/test_function_app.py +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -4,6 +4,10 @@ from pytest import fixture from data_safe_haven.resources.gitea_mirror.functions.function_app import ( + gitea_host, + api_root, + migrate_path, + repos_path, str_or_none, create_mirror, delete_mirror, @@ -40,11 +44,11 @@ def test_create_mirror(self, create_mirror_func, requests_mock): ) requests_mock.post( - "http://localhost:3000/api/v1/repos/migrate", + gitea_host + api_root + migrate_path, status_code=201 ) requests_mock.patch( - "http://localhost:3000/api/v1/repos/username/repo", + gitea_host + api_root + repos_path + "/username/repo", status_code=200 ) @@ -64,11 +68,11 @@ def test_create_mirror_missing_args(self, create_mirror_func, requests_mock): ) requests_mock.post( - "http://localhost:3000/api/v1/repos/migrate", + gitea_host + api_root + migrate_path, status_code=201 ) requests_mock.patch( - "http://localhost:3000/api/v1/repos/username/repo", + gitea_host + api_root + repos_path + "/username/repo", status_code=200 ) @@ -89,11 +93,11 @@ def test_create_mirror_mirror_fail(self, create_mirror_func, requests_mock): ) requests_mock.post( - "http://localhost:3000/api/v1/repos/migrate", + gitea_host + api_root + migrate_path, status_code=409 ) requests_mock.patch( - "http://localhost:3000/api/v1/repos/username/repo", + gitea_host + api_root + repos_path + "/username/repo", status_code=200 ) @@ -114,11 +118,11 @@ def test_create_mirror_configure_fail(self, create_mirror_func, requests_mock): ) requests_mock.post( - "http://localhost:3000/api/v1/repos/migrate", + gitea_host + api_root + migrate_path, status_code=201 ) requests_mock.patch( - "http://localhost:3000/api/v1/repos/username/repo", + gitea_host + api_root + repos_path + "/username/repo", status_code=403 ) @@ -141,7 +145,7 @@ def test_delete_mirror(self, delete_mirror_func, requests_mock): ) requests_mock.delete( - "http://localhost:3000/api/v1/repos/admin/repo", + gitea_host + api_root + repos_path + "/admin/repo", status_code=204, ) @@ -160,7 +164,7 @@ def test_delete_mirror_missing_args(self, delete_mirror_func, requests_mock): ) requests_mock.delete( - "http://localhost:3000/api/v1/repos/admin/repo", + gitea_host + api_root + repos_path + "/admin/repo", status_code=204, ) @@ -181,7 +185,7 @@ def test_delete_mirror_fail(self, delete_mirror_func, requests_mock): ) requests_mock.delete( - "http://localhost:3000/api/v1/repos/admin/repo", + gitea_host + api_root + repos_path + "/admin/repo", status_code=404, ) From e9c8f290adcb5d250740640ada0d437b877ba895 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 11 Jul 2024 15:58:48 +0100 Subject: [PATCH 28/44] Run lint:fmt --- .../functions/test_function_app.py | 132 +++++++++--------- 1 file changed, 65 insertions(+), 67 deletions(-) diff --git a/tests/resources/gitea_mirror/functions/test_function_app.py b/tests/resources/gitea_mirror/functions/test_function_app.py index 3eb8038fd4..10169703da 100644 --- a/tests/resources/gitea_mirror/functions/test_function_app.py +++ b/tests/resources/gitea_mirror/functions/test_function_app.py @@ -4,13 +4,13 @@ from pytest import fixture from data_safe_haven.resources.gitea_mirror.functions.function_app import ( - gitea_host, api_root, + create_mirror, + delete_mirror, + gitea_host, migrate_path, repos_path, str_or_none, - create_mirror, - delete_mirror, ) @@ -34,22 +34,20 @@ class TestCreateMirror: def test_create_mirror(self, create_mirror_func, requests_mock): req = func.HttpRequest( method="POST", - body=json.dumps({ - "address": "https://github.com/user/repo", - "name": "repo", - "password": "password", - "username": "username", - }).encode(), + body=json.dumps( + { + "address": "https://github.com/user/repo", + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), url="/api/create-mirror", ) - requests_mock.post( - gitea_host + api_root + migrate_path, - status_code=201 - ) + requests_mock.post(gitea_host + api_root + migrate_path, status_code=201) requests_mock.patch( - gitea_host + api_root + repos_path + "/username/repo", - status_code=200 + gitea_host + api_root + repos_path + "/username/repo", status_code=200 ) response = create_mirror_func(req) @@ -59,21 +57,19 @@ def test_create_mirror(self, create_mirror_func, requests_mock): def test_create_mirror_missing_args(self, create_mirror_func, requests_mock): req = func.HttpRequest( method="POST", - body=json.dumps({ - "name": "repo", - "password": "password", - "username": "username", - }).encode(), + body=json.dumps( + { + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), url="/api/create-mirror", ) - requests_mock.post( - gitea_host + api_root + migrate_path, - status_code=201 - ) + requests_mock.post(gitea_host + api_root + migrate_path, status_code=201) requests_mock.patch( - gitea_host + api_root + repos_path + "/username/repo", - status_code=200 + gitea_host + api_root + repos_path + "/username/repo", status_code=200 ) response = create_mirror_func(req) @@ -83,22 +79,20 @@ def test_create_mirror_missing_args(self, create_mirror_func, requests_mock): def test_create_mirror_mirror_fail(self, create_mirror_func, requests_mock): req = func.HttpRequest( method="POST", - body=json.dumps({ - "address": "https://github.com/user/repo", - "name": "repo", - "password": "password", - "username": "username", - }).encode(), + body=json.dumps( + { + "address": "https://github.com/user/repo", + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), url="/api/create-mirror", ) - requests_mock.post( - gitea_host + api_root + migrate_path, - status_code=409 - ) + requests_mock.post(gitea_host + api_root + migrate_path, status_code=409) requests_mock.patch( - gitea_host + api_root + repos_path + "/username/repo", - status_code=200 + gitea_host + api_root + repos_path + "/username/repo", status_code=200 ) response = create_mirror_func(req) @@ -108,22 +102,20 @@ def test_create_mirror_mirror_fail(self, create_mirror_func, requests_mock): def test_create_mirror_configure_fail(self, create_mirror_func, requests_mock): req = func.HttpRequest( method="POST", - body=json.dumps({ - "address": "https://github.com/user/repo", - "name": "repo", - "password": "password", - "username": "username", - }).encode(), + body=json.dumps( + { + "address": "https://github.com/user/repo", + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), url="/api/create-mirror", ) - requests_mock.post( - gitea_host + api_root + migrate_path, - status_code=201 - ) + requests_mock.post(gitea_host + api_root + migrate_path, status_code=201) requests_mock.patch( - gitea_host + api_root + repos_path + "/username/repo", - status_code=403 + gitea_host + api_root + repos_path + "/username/repo", status_code=403 ) response = create_mirror_func(req) @@ -135,12 +127,14 @@ class TestDeleteMirror: def test_delete_mirror(self, delete_mirror_func, requests_mock): req = func.HttpRequest( method="POST", - body=json.dumps({ - "owner": "admin", - "name": "repo", - "password": "password", - "username": "username", - }).encode(), + body=json.dumps( + { + "owner": "admin", + "name": "repo", + "password": "password", + "username": "username", + } + ).encode(), url="/api/delete-mirror", ) @@ -155,11 +149,13 @@ def test_delete_mirror(self, delete_mirror_func, requests_mock): def test_delete_mirror_missing_args(self, delete_mirror_func, requests_mock): req = func.HttpRequest( method="POST", - body=json.dumps({ - "name": "repo", - "owner": "admin", - "password": "password", - }).encode(), + body=json.dumps( + { + "name": "repo", + "owner": "admin", + "password": "password", + } + ).encode(), url="/api/delete-mirror", ) @@ -175,12 +171,14 @@ def test_delete_mirror_missing_args(self, delete_mirror_func, requests_mock): def test_delete_mirror_fail(self, delete_mirror_func, requests_mock): req = func.HttpRequest( method="POST", - body=json.dumps({ - "name": "repo", - "owner": "admin", - "password": "password", - "username": "admin", - }).encode(), + body=json.dumps( + { + "name": "repo", + "owner": "admin", + "password": "password", + "username": "admin", + } + ).encode(), url="/api/delete-mirror", ) From 9cf8350a4d19f840fd4b5fb4f2169cabf99fe7b0 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 12 Jul 2024 12:02:52 +0100 Subject: [PATCH 29/44] WIP: Add app service infrastructure --- .../infrastructure/programs/sre/apps.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 data_safe_haven/infrastructure/programs/sre/apps.py diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py new file mode 100644 index 0000000000..acd8424db9 --- /dev/null +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -0,0 +1,109 @@ +"""Pulumi component for SRE function/web apps""" + +from typing import Mapping + +from pulumi import ComponentResource, ResourceOptions, Input, FileArchive +from pulumi_azure_native import ( + resources, + storage, + web, +) + +from data_safe_haven.functions import ( + alphanumeric, + truncate_tokens, +) +from data_safe_haven.resources import resources_path + + +class SREAppsProps: + """Properties for SREAppsComponent""" + pass + + +class SREAppsComponent(ComponentResource): + """Deploy SRE function/web apps with Pulumi""" + + def __init__( + self, + name: str, + stack_name: str, + props: SREAppsProps, + opts: ResourceOptions | None = None, + tags: Input[Mapping[str, Input[str]]] | None = None, + ) -> None: + super().__init__("dsh:sre:AppsComponent", name, {}, opts) + child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) + child_tags = tags if tags else {} + + # Deploy resource group + resource_group = resources.ResourceGroup( + f"{self._name}_resource_group", + location=props.location, + resource_group_name=f"{stack_name}-rg-apps", + opts=child_opts, + tags=child_tags, + ) + + # Deploy storage account + # The storage account holds app data/configuration + storage_account = storage.StorageAccount( + f"{self._name}_storage_account", + account_name=alphanumeric( + f"{''.join(truncate_tokens(stack_name.split('-'), 14))}apps" + )[:24], + kind=storage.Kind.STORAGE_V2, + location=props.location, + resource_group_name=resource_group.name, + sku=storage.SkuArgs(name=storage.SkuName.STANDARD_GRS), + opts=child_opts, + tags=child_tags, + ) + + # Create function apps container + container = storage.BlobContainer( + f"{self._name}_container_functions", + account_name=storage_account.name, + container_name="functions", + public_access=storage.PublicAccess.NONE, + resource_group_name=resource_group.name, + opts=ResourceOptions.merge( + child_opts, + ResourceOptions(parent=storage_account), + ), + tags=child_tags, + ) + + # Upload Gitea mirror function app + blob_gitea_mirror = storage.Blob( + f"{self._name}_blob_gitea_mirror", + account_name=storage_account.name, + container_name=container.name, + resource_group_name=resource_group.name, + source=FileArchive( + str((resources_path / "gitea_mirror" / "functions").absolute()), + ), + opts=ResourceOptions.merge( + child_opts, + ResourceOptions(parent=container), + ), + tags=child_tags, + ) + + # Deploy service plan + app_service_plan = web.AppServicePlan( + f"{self._name}_app_service_plan", + kind="linux", + location=props.location, + name=f"{stack_name}-app-service-plan", + resource_group_name=resource_group.name, + sku={ + "name": "B1", + "tier": "Basic", + "size": "B1", + "family": "B", + "capacity": 1 + } + ) + + # connection_string = get_connection_string() From a91f17ffd07412a5b66a67684e421e85692a1b4e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 30 Jul 2024 11:21:30 +0100 Subject: [PATCH 30/44] WIP remove resource group --- .../infrastructure/programs/sre/apps.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index acd8424db9..003e13627b 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -1,8 +1,8 @@ """Pulumi component for SRE function/web apps""" -from typing import Mapping +from collections.abc import Mapping -from pulumi import ComponentResource, ResourceOptions, Input, FileArchive +from pulumi import ComponentResource, FileArchive, Input, Output, ResourceOptions from pulumi_azure_native import ( resources, storage, @@ -13,12 +13,22 @@ alphanumeric, truncate_tokens, ) +from data_safe_haven.infrastructure.common import ( + get_name_from_rg, +) from data_safe_haven.resources import resources_path class SREAppsProps: """Properties for SREAppsComponent""" - pass + + def __init__( + self, + resource_group: Input[resources.ResourceGroup], + ): + self.resource_group_name = Output.from_input(resource_group).apply( + get_name_from_rg + ) class SREAppsComponent(ComponentResource): @@ -36,15 +46,6 @@ def __init__( child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) child_tags = tags if tags else {} - # Deploy resource group - resource_group = resources.ResourceGroup( - f"{self._name}_resource_group", - location=props.location, - resource_group_name=f"{stack_name}-rg-apps", - opts=child_opts, - tags=child_tags, - ) - # Deploy storage account # The storage account holds app data/configuration storage_account = storage.StorageAccount( @@ -54,7 +55,7 @@ def __init__( )[:24], kind=storage.Kind.STORAGE_V2, location=props.location, - resource_group_name=resource_group.name, + resource_group_name=props.resource_group_name, sku=storage.SkuArgs(name=storage.SkuName.STANDARD_GRS), opts=child_opts, tags=child_tags, @@ -66,7 +67,7 @@ def __init__( account_name=storage_account.name, container_name="functions", public_access=storage.PublicAccess.NONE, - resource_group_name=resource_group.name, + resource_group_name=props.resource_group_name, opts=ResourceOptions.merge( child_opts, ResourceOptions(parent=storage_account), @@ -79,7 +80,7 @@ def __init__( f"{self._name}_blob_gitea_mirror", account_name=storage_account.name, container_name=container.name, - resource_group_name=resource_group.name, + resource_group_name=props.resource_group_name, source=FileArchive( str((resources_path / "gitea_mirror" / "functions").absolute()), ), @@ -96,14 +97,14 @@ def __init__( kind="linux", location=props.location, name=f"{stack_name}-app-service-plan", - resource_group_name=resource_group.name, + resource_group_name=props.resource_group_name, sku={ "name": "B1", "tier": "Basic", "size": "B1", "family": "B", - "capacity": 1 - } + "capacity": 1, + }, ) # connection_string = get_connection_string() From e8be73a515dbcb383d1e03cee45254809ad250f3 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 30 Jul 2024 14:48:07 +0100 Subject: [PATCH 31/44] WIP: Add webapp --- .../infrastructure/programs/sre/apps.py | 73 ++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index 003e13627b..5b37a3e962 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -24,8 +24,10 @@ class SREAppsProps: def __init__( self, + location: Input[str], resource_group: Input[resources.ResourceGroup], ): + self.location = location self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg ) @@ -91,6 +93,14 @@ def __init__( tags=child_tags, ) + # Get URL of app blob + blob_url = get_blob_url( + blob=blob_gitea_mirror, + container=container, + storage_account=storage_account, + resource_group_name=props.resource_group_name, + ) + # Deploy service plan app_service_plan = web.AppServicePlan( f"{self._name}_app_service_plan", @@ -107,4 +117,65 @@ def __init__( }, ) - # connection_string = get_connection_string() + # Deploy app + web.WebApp( + f"{self._name}_web_app", + enabled=True, + https_only=True, + kind="FunctionApp", + location=props.location, + name="giteamirror", + resource_group_name=props.resource_group_name, + server_farm_id=app_service_plan.id, + site_config=web.SiteConfig( + app_settings=[ + {"name": "runtime", "value": "python"}, + {"name": "FUNCTIONS_WORKER_RUNTIME", "value": "python"}, + {"name": "WEBSITE_RUN_FROM_PACKAGE", "value": blob_url}, + {"name": "FUNCTIONS_EXTENSION_VERSION", "value": "~4"}, + ], + ) + ) + + +def get_blob_url( + blob: Input[storage.Blob], + container: Input[storage.BlobContainer], + storage_account: Input[storage.StorageAccount], + resource_group_name: Input[str], +) -> Output[str]: + sas = storage.list_storage_account_service_sas_output( + account_name=storage_account.name, + protocols=storage.HttpProtocol.Https, + # shared_access_expiry_time="2030-01-01", + # shared_access_start_time="2021-01-01", + resource_group_name=resource_group_name, + # Access to container + resource=storage.SignedResource.C, + # Read access + permissions=storage.Permissions.R, + canonicalized_resource=Output.format( + "/blob/{account_name}/{container_name}", + account_name=storage_account.name, + container_name=container.name, + ), + content_type="application/json", + cache_control="max-age=5", + content_disposition="inline", + content_encoding="deflate", + ) + token = sas.service_sas_token + # return Output.format( + # "https://{0}.blob.core.windows.net/{1}/{2}?{3}", + # storage_account.name, + # container.name, + # blob.name, + # token, + # ) + return Output.format( + "https://{storage_account_name}.blob.core.windows.net/{container_name}/{blob_name}?{token}", + storage_account_name=storage_account.name, + container_name=container.name, + blob_name=blob.name, + token=token, + ) From b5cc5e11f40c48b703cd920637d4df2f95b5827c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 30 Jul 2024 14:54:55 +0100 Subject: [PATCH 32/44] Fix linting --- data_safe_haven/infrastructure/programs/sre/apps.py | 2 +- typings/pulumi/__init__.pyi | 3 ++- typings/pulumi_azure_native/__init__.pyi | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index 5b37a3e962..384730a830 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -134,7 +134,7 @@ def __init__( {"name": "WEBSITE_RUN_FROM_PACKAGE", "value": blob_url}, {"name": "FUNCTIONS_EXTENSION_VERSION", "value": "~4"}, ], - ) + ), ) diff --git a/typings/pulumi/__init__.pyi b/typings/pulumi/__init__.pyi index e1468220dd..b8e13de013 100644 --- a/typings/pulumi/__init__.pyi +++ b/typings/pulumi/__init__.pyi @@ -1,6 +1,6 @@ import pulumi.automation as automation import pulumi.dynamic as dynamic -from pulumi.asset import FileAsset +from pulumi.asset import FileArchive, FileAsset from pulumi.config import ( Config, ) @@ -22,6 +22,7 @@ __all__ = [ "Config", "dynamic", "export", + "FileArchive", "FileAsset", "Input", "Output", diff --git a/typings/pulumi_azure_native/__init__.pyi b/typings/pulumi_azure_native/__init__.pyi index 56be0a1e3a..bf031a21c7 100644 --- a/typings/pulumi_azure_native/__init__.pyi +++ b/typings/pulumi_azure_native/__init__.pyi @@ -15,6 +15,7 @@ import pulumi_azure_native.operationsmanagement as operationsmanagement import pulumi_azure_native.resources as resources import pulumi_azure_native.sql as sql import pulumi_azure_native.storage as storage +import pulumi_azure_native.web as web __all__ = [ "automation", @@ -33,5 +34,6 @@ __all__ = [ "resources", "sql", "storage", + "web", "_utilities", ] From fd82175b17438bed2eb51ebd669ee89c6a76ff82 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 30 Jul 2024 15:04:17 +0100 Subject: [PATCH 33/44] Add apps component to sre --- .../infrastructure/programs/declarative_sre.py | 15 +++++++++++++++ .../infrastructure/programs/sre/apps.py | 10 ++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 1698be923b..0262f4ed64 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -2,6 +2,10 @@ import pulumi from pulumi_azure_native import resources +from sre.apps import ( + SREAppsComponent, + SREAppsProps, +) from data_safe_haven.config import Context, SREConfig from data_safe_haven.infrastructure.common import DockerHubCredentials @@ -327,6 +331,17 @@ def __call__(self) -> None: tags=self.tags, ) + # Deploy apps + SREAppsComponent( + "sre_apps", + self.stack_name, + SREAppsProps( + location=self.config.azure.location, + resource_group_name=resource_group.name, + ), + tags=self.tags, + ) + # Deploy monitoring monitoring = SREMonitoringComponent( "sre_monitoring", diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index 384730a830..f3160f46ae 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -4,7 +4,6 @@ from pulumi import ComponentResource, FileArchive, Input, Output, ResourceOptions from pulumi_azure_native import ( - resources, storage, web, ) @@ -13,9 +12,6 @@ alphanumeric, truncate_tokens, ) -from data_safe_haven.infrastructure.common import ( - get_name_from_rg, -) from data_safe_haven.resources import resources_path @@ -25,12 +21,10 @@ class SREAppsProps: def __init__( self, location: Input[str], - resource_group: Input[resources.ResourceGroup], + resource_group_name: Input[str], ): self.location = location - self.resource_group_name = Output.from_input(resource_group).apply( - get_name_from_rg - ) + self.resource_group_name = resource_group_name class SREAppsComponent(ComponentResource): From 0b6ba2991fd4fffcb5c27d2fbcc165ce691b5b6a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 30 Jul 2024 15:07:06 +0100 Subject: [PATCH 34/44] Fix import --- .../infrastructure/programs/declarative_sre.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 0262f4ed64..2307bf3d7e 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -2,10 +2,6 @@ import pulumi from pulumi_azure_native import resources -from sre.apps import ( - SREAppsComponent, - SREAppsProps, -) from data_safe_haven.config import Context, SREConfig from data_safe_haven.infrastructure.common import DockerHubCredentials @@ -14,6 +10,10 @@ SREApplicationGatewayComponent, SREApplicationGatewayProps, ) +from .sre.apps import ( + SREAppsComponent, + SREAppsProps, +) from .sre.apt_proxy_server import SREAptProxyServerComponent, SREAptProxyServerProps from .sre.backup import ( SREBackupComponent, From 491a27a3982881b2d997541b4292c075ceca9ada Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 31 Jul 2024 09:40:49 +0100 Subject: [PATCH 35/44] Correct tags and HTTPS enum --- data_safe_haven/infrastructure/programs/sre/apps.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index f3160f46ae..86f3f824e8 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -68,7 +68,6 @@ def __init__( child_opts, ResourceOptions(parent=storage_account), ), - tags=child_tags, ) # Upload Gitea mirror function app @@ -84,7 +83,6 @@ def __init__( child_opts, ResourceOptions(parent=container), ), - tags=child_tags, ) # Get URL of app blob @@ -109,6 +107,7 @@ def __init__( "family": "B", "capacity": 1, }, + tags=child_tags, ) # Deploy app @@ -129,6 +128,7 @@ def __init__( {"name": "FUNCTIONS_EXTENSION_VERSION", "value": "~4"}, ], ), + tags=child_tags, ) @@ -140,7 +140,7 @@ def get_blob_url( ) -> Output[str]: sas = storage.list_storage_account_service_sas_output( account_name=storage_account.name, - protocols=storage.HttpProtocol.Https, + protocols=storage.HttpProtocol.HTTPS, # shared_access_expiry_time="2030-01-01", # shared_access_start_time="2021-01-01", resource_group_name=resource_group_name, From 0753fa1a40eefe7d6f038456a4787784cdced3b2 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 31 Jul 2024 11:11:39 +0100 Subject: [PATCH 36/44] Fix arguments --- .../infrastructure/programs/sre/apps.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index 86f3f824e8..3c92199970 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -117,10 +117,10 @@ def __init__( https_only=True, kind="FunctionApp", location=props.location, - name="giteamirror", + name=f"{stack_name}gitea-mirror-api", resource_group_name=props.resource_group_name, server_farm_id=app_service_plan.id, - site_config=web.SiteConfig( + site_config=web.SiteConfigArgs( app_settings=[ {"name": "runtime", "value": "python"}, {"name": "FUNCTIONS_WORKER_RUNTIME", "value": "python"}, @@ -141,8 +141,8 @@ def get_blob_url( sas = storage.list_storage_account_service_sas_output( account_name=storage_account.name, protocols=storage.HttpProtocol.HTTPS, - # shared_access_expiry_time="2030-01-01", - # shared_access_start_time="2021-01-01", + shared_access_expiry_time="2030-01-01", + shared_access_start_time="2021-01-01", resource_group_name=resource_group_name, # Access to container resource=storage.SignedResource.C, @@ -159,13 +159,6 @@ def get_blob_url( content_encoding="deflate", ) token = sas.service_sas_token - # return Output.format( - # "https://{0}.blob.core.windows.net/{1}/{2}?{3}", - # storage_account.name, - # container.name, - # blob.name, - # token, - # ) return Output.format( "https://{storage_account_name}.blob.core.windows.net/{container_name}/{blob_name}?{token}", storage_account_name=storage_account.name, From 7fdc5fb29341243087c12a1e9351c8411d99d079 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 31 Jul 2024 11:12:54 +0100 Subject: [PATCH 37/44] Change Function App name --- data_safe_haven/infrastructure/programs/sre/apps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index 3c92199970..2d566151de 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -117,7 +117,7 @@ def __init__( https_only=True, kind="FunctionApp", location=props.location, - name=f"{stack_name}gitea-mirror-api", + name=f"{stack_name}-gitea-mirror-api", resource_group_name=props.resource_group_name, server_farm_id=app_service_plan.id, site_config=web.SiteConfigArgs( From 8dc0356831feac281808be41ef0c4f3ff40880cb Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 31 Jul 2024 11:31:17 +0100 Subject: [PATCH 38/44] Update WebApp type --- data_safe_haven/infrastructure/programs/sre/apps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index 2d566151de..ec094bb4fe 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -99,6 +99,7 @@ def __init__( kind="linux", location=props.location, name=f"{stack_name}-app-service-plan", + reserved=True, resource_group_name=props.resource_group_name, sku={ "name": "B1", @@ -115,7 +116,7 @@ def __init__( f"{self._name}_web_app", enabled=True, https_only=True, - kind="FunctionApp", + kind="functionapp,linux", location=props.location, name=f"{stack_name}-gitea-mirror-api", resource_group_name=props.resource_group_name, From e495c6648cb31661ec1b40deb4fa2068bed55c60 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 1 Aug 2024 10:46:42 +0100 Subject: [PATCH 39/44] WIP: Add connection string --- .../infrastructure/programs/sre/apps.py | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index ec094bb4fe..4376b0a37f 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -93,6 +93,12 @@ def __init__( resource_group_name=props.resource_group_name, ) + # Get connection string + connection_string = get_connection_string( + resource_group_name=props.resource_group_name, + storage_account=storage_account, + ) + # Deploy service plan app_service_plan = web.AppServicePlan( f"{self._name}_app_service_plan", @@ -123,7 +129,9 @@ def __init__( server_farm_id=app_service_plan.id, site_config=web.SiteConfigArgs( app_settings=[ + {"name": "AzureWebJobsStorage", "value": connection_string}, {"name": "runtime", "value": "python"}, + {"name": "pythonVersion", "value": "3.11"}, {"name": "FUNCTIONS_WORKER_RUNTIME", "value": "python"}, {"name": "WEBSITE_RUN_FROM_PACKAGE", "value": blob_url}, {"name": "FUNCTIONS_EXTENSION_VERSION", "value": "~4"}, @@ -136,8 +144,8 @@ def __init__( def get_blob_url( blob: Input[storage.Blob], container: Input[storage.BlobContainer], - storage_account: Input[storage.StorageAccount], resource_group_name: Input[str], + storage_account: Input[storage.StorageAccount], ) -> Output[str]: sas = storage.list_storage_account_service_sas_output( account_name=storage_account.name, @@ -154,10 +162,10 @@ def get_blob_url( account_name=storage_account.name, container_name=container.name, ), - content_type="application/json", - cache_control="max-age=5", - content_disposition="inline", - content_encoding="deflate", + # content_type="application/json", + # cache_control="max-age=5", + # content_disposition="inline", + # content_encoding="deflate", ) token = sas.service_sas_token return Output.format( @@ -167,3 +175,20 @@ def get_blob_url( blob_name=blob.name, token=token, ) + + +def get_connection_string( + resource_group_name: Input[str], + storage_account: Input[storage.StorageAccount], +) -> Output[str]: + storage_account_keys = storage.list_storage_account_keys_output( + resource_group_name=resource_group_name, + account_name=storage_account.name, + ) + primary_storage_key = storage_account_keys.keys[0].value + + return Output.format( + "DefaultEndpointsProtocol=https;AccountName={storage_account_name};AccountKey={primary_storage_key}", + storage_account_name=storage_account.name, + primary_storage_key=primary_storage_key, + ) From 6dffae9059bce35157342d4e0a1d79ae008a6287 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 1 Aug 2024 16:21:18 +0100 Subject: [PATCH 40/44] Set webapp to always on --- data_safe_haven/infrastructure/programs/sre/apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index 4376b0a37f..f637548e16 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -128,6 +128,7 @@ def __init__( resource_group_name=props.resource_group_name, server_farm_id=app_service_plan.id, site_config=web.SiteConfigArgs( + always_on=True, app_settings=[ {"name": "AzureWebJobsStorage", "value": connection_string}, {"name": "runtime", "value": "python"}, From 5aec4e5b5d07678019fc8b2169e82ddbf4d672aa Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 1 Aug 2024 16:21:40 +0100 Subject: [PATCH 41/44] Set Python version --- data_safe_haven/infrastructure/programs/sre/apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/infrastructure/programs/sre/apps.py b/data_safe_haven/infrastructure/programs/sre/apps.py index f637548e16..40b6346fe4 100644 --- a/data_safe_haven/infrastructure/programs/sre/apps.py +++ b/data_safe_haven/infrastructure/programs/sre/apps.py @@ -137,6 +137,7 @@ def __init__( {"name": "WEBSITE_RUN_FROM_PACKAGE", "value": blob_url}, {"name": "FUNCTIONS_EXTENSION_VERSION", "value": "~4"}, ], + linux_fx_version="Python|3.11", ), tags=child_tags, ) From 97e4ca8836a1a45fa6f6a940d1fc333ff3aba022 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 1 Aug 2024 16:22:34 +0100 Subject: [PATCH 42/44] Restructure using v1 programming model --- .../functions/create_mirror/__init__.py | 81 +++++++++ .../functions/create_mirror/function.json | 20 ++ .../functions/delete_mirror/__init__.py | 47 +++++ .../functions/delete_mirror/function.json | 20 ++ .../gitea_mirror/functions/function_app.py | 171 ------------------ .../gitea_mirror/functions/host.json | 16 +- .../gitea_mirror/functions/requirements.txt | 6 - .../functions/shared_code/__init__.py | 60 ++++++ 8 files changed, 232 insertions(+), 189 deletions(-) create mode 100644 data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py create mode 100644 data_safe_haven/resources/gitea_mirror/functions/create_mirror/function.json create mode 100644 data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py create mode 100644 data_safe_haven/resources/gitea_mirror/functions/delete_mirror/function.json delete mode 100644 data_safe_haven/resources/gitea_mirror/functions/function_app.py create mode 100644 data_safe_haven/resources/gitea_mirror/functions/shared_code/__init__.py diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py new file mode 100644 index 0000000000..47e5b26f56 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py @@ -0,0 +1,81 @@ +import logging + +import azure.functions as func +import requests +from requests.auth import HTTPBasicAuth + +from shared_code import ( + get_args, check_args, missing_parameters_repsonse, timeout, gitea_host, api_root, + migrate_path, repos_path, handle_response +) + + +def main(req: func.HttpRequest) -> func.HttpResponse: + logging.info("Request received.") + + raw_args = get_args( + [ + "address", + "name", + "password", + "username", + ], + req, + ) + args = check_args(raw_args) + if not args: + return missing_parameters_repsonse() + + extra_data = { + "description": f"Read-only mirror of {args['address']}", + "mirror": True, + "mirror_interval": "10m", + } + + auth = HTTPBasicAuth( + username=args["username"], + password=args["password"], + ) + + logging.info("Sending request to create mirror.") + + response = requests.post( + auth=auth, + data={ + "clone_addr": args["address"], + "repo_name": args["name"], + } + | extra_data, + timeout=timeout, + url=gitea_host + api_root + migrate_path, + ) + + if r := handle_response(response, [201], "Error creating repository."): + return r + + # Some arguments of the migrate endpoint seem to be ignored or overwritten. + # We set repository settings here. + logging.info("Sending request to configure mirror repo.") + + response = requests.patch( + auth=auth, + data={ + "has_actions": False, + "has_issues": False, + "has_packages": False, + "has_projects": False, + "has_pull_requests": False, + "has_releases": False, + "has_wiki": False, + }, + timeout=timeout, + url=gitea_host + api_root + repos_path + f"/{args['username']}/{args['name']}", + ) + + if r := handle_response(response, [200], "Error configuring repository."): + return r + + return func.HttpResponse( + "Mirror successfully created.", + status_code=200, + ) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function.json b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function.json new file mode 100644 index 0000000000..4667f0aca9 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py new file mode 100644 index 0000000000..c003015e04 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py @@ -0,0 +1,47 @@ +import logging + +import azure.functions as func +import requests +from requests.auth import HTTPBasicAuth + +from shared_code import ( + get_args, check_args, missing_parameters_repsonse, timeout, gitea_host, api_root, + repos_path, handle_response +) + + +def main(req: func.HttpRequest) -> func.HttpResponse: + logging.info("Request received.") + + raw_args = get_args( + [ + "name", + "owner", + "password", + "username", + ], + req, + ) + args = check_args(raw_args) + if not args: + return missing_parameters_repsonse() + + auth = HTTPBasicAuth( + username=args["username"], + password=args["password"], + ) + + logging.info("Sending request to delete repository.") + response = requests.delete( + auth=auth, + timeout=timeout, + url=gitea_host + api_root + repos_path + f"/{args['owner']}/{args['name']}", + ) + + if r := handle_response(response, [204], "Error deleting repository."): + return r + + return func.HttpResponse( + "Repository successfully deleted.", + status_code=200, + ) diff --git a/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/function.json b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/function.json new file mode 100644 index 0000000000..4667f0aca9 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/function.json @@ -0,0 +1,20 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "authLevel": "anonymous", + "type": "httpTrigger", + "direction": "in", + "name": "req", + "methods": [ + "get", + "post" + ] + }, + { + "type": "http", + "direction": "out", + "name": "$return" + } + ] +} \ No newline at end of file diff --git a/data_safe_haven/resources/gitea_mirror/functions/function_app.py b/data_safe_haven/resources/gitea_mirror/functions/function_app.py deleted file mode 100644 index 28410913d6..0000000000 --- a/data_safe_haven/resources/gitea_mirror/functions/function_app.py +++ /dev/null @@ -1,171 +0,0 @@ -import logging -from typing import Any - -import azure.functions as func -import requests -from requests.auth import HTTPBasicAuth - -app = func.FunctionApp() - -# Global parameters -# gitea_host = "http://gitea_mirror.local" -gitea_host = "http://localhost:3000" -api_root = "/api/v1" -migrate_path = "/repos/migrate" -repos_path = "/repos" -timeout = 60 - - -def get_args(args: list[str], req: func.HttpRequest) -> dict[str, str | None]: - try: - req_body = req.get_json() - except ValueError: - return {} - - args_dict = {arg: str_or_none(req_body.get(arg)) for arg in args} - logging.info(f"Parameters: {args}.") - return args_dict - - -def str_or_none(item: Any) -> str | None: - return str(item) if item is not None else None - - -def check_args(args: dict[str, str | None]) -> dict[str, str] | None: - if None in args.values(): - return None - else: - return {key: str(value) for key, value in args.items()} - - -def missing_parameters_repsonse() -> func.HttpResponse: - msg = "Required parameter not provided." - logging.critical(msg) - return func.HttpResponse( - msg, - status_code=400, - ) - - -def handle_response( - response: requests.Response, valid_codes: list[int], error_message: str -) -> func.HttpResponse | None: - logging.info(f"Response status code: {response.status_code}.") - logging.debug(f"Response contents: {response.text}.") - if response.status_code not in valid_codes: - return func.HttpResponse( - error_message, - status_code=400, - ) - else: - return None - - -@app.route(route="create-mirror", auth_level=func.AuthLevel.ANONYMOUS) -def create_mirror(req: func.HttpRequest) -> func.HttpResponse: - logging.info("Request received.") - - raw_args = get_args( - [ - "address", - "name", - "password", - "username", - ], - req, - ) - args = check_args(raw_args) - if not args: - return missing_parameters_repsonse() - - extra_data = { - "description": f"Read-only mirror of {args['address']}", - "mirror": True, - "mirror_interval": "10m", - } - - auth = HTTPBasicAuth( - username=args["username"], - password=args["password"], - ) - - logging.info("Sending request to create mirror.") - - response = requests.post( - auth=auth, - data={ - "clone_addr": args["address"], - "repo_name": args["name"], - } - | extra_data, - timeout=timeout, - url=gitea_host + api_root + migrate_path, - ) - - if r := handle_response(response, [201], "Error creating repository."): - return r - - # Some arguments of the migrate endpoint seem to be ignored or overwritten. - # We set repository settings here. - logging.info("Sending request to configure mirror repo.") - - response = requests.patch( - auth=auth, - data={ - "has_actions": False, - "has_issues": False, - "has_packages": False, - "has_projects": False, - "has_pull_requests": False, - "has_releases": False, - "has_wiki": False, - }, - timeout=timeout, - url=gitea_host + api_root + repos_path + f"/{args['username']}/{args['name']}", - ) - - if r := handle_response(response, [200], "Error configuring repository."): - return r - - return func.HttpResponse( - "Mirror successfully created.", - status_code=200, - ) - - -@app.route(route="delete-mirror", auth_level=func.AuthLevel.ANONYMOUS) -def delete_mirror(req: func.HttpRequest) -> func.HttpResponse: - logging.info("Request received.") - - raw_args = get_args( - [ - "name", - "owner", - "password", - "username", - ], - req, - ) - args = check_args(raw_args) - if not args: - return missing_parameters_repsonse() - - auth = HTTPBasicAuth( - username=args["username"], - password=args["password"], - ) - - logging.info("Sending request to delete repository.") - response = requests.delete( - auth=auth, - timeout=timeout, - url=gitea_host + api_root + repos_path + f"/{args['owner']}/{args['name']}", - ) - - if r := handle_response(response, [204], "Error deleting repository."): - return r - - return func.HttpResponse( - "Repository successfully deleted.", - status_code=200, - ) diff --git a/data_safe_haven/resources/gitea_mirror/functions/host.json b/data_safe_haven/resources/gitea_mirror/functions/host.json index 06d01bdaa9..221cb15341 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/host.json +++ b/data_safe_haven/resources/gitea_mirror/functions/host.json @@ -1,15 +1,7 @@ { - "version": "2.0", - "logging": { - "applicationInsights": { - "samplingSettings": { - "isEnabled": true, - "excludedTypes": "Request" - } + "version": "2.0", + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" } - }, - "extensionBundle": { - "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[4.*, 5.0.0)" - } } diff --git a/data_safe_haven/resources/gitea_mirror/functions/requirements.txt b/data_safe_haven/resources/gitea_mirror/functions/requirements.txt index b1f67106a3..e69de29bb2 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/requirements.txt +++ b/data_safe_haven/resources/gitea_mirror/functions/requirements.txt @@ -1,6 +0,0 @@ -# Do not include azure-functions-worker in this file -# The Python Worker is managed by the Azure Functions platform -# Manually managing azure-functions-worker may cause unexpected issues - -azure-functions -requests diff --git a/data_safe_haven/resources/gitea_mirror/functions/shared_code/__init__.py b/data_safe_haven/resources/gitea_mirror/functions/shared_code/__init__.py new file mode 100644 index 0000000000..557c93fce3 --- /dev/null +++ b/data_safe_haven/resources/gitea_mirror/functions/shared_code/__init__.py @@ -0,0 +1,60 @@ +import logging +from typing import Any + +import azure.functions as func +import requests + +app = func.FunctionApp() + +# Global parameters +# gitea_host = "http://gitea_mirror.local" +gitea_host = "http://localhost:3000" +api_root = "/api/v1" +migrate_path = "/repos/migrate" +repos_path = "/repos" +timeout = 60 + + +def get_args(args: list[str], req: func.HttpRequest) -> dict[str, str | None]: + try: + req_body = req.get_json() + except ValueError: + return {} + + args_dict = {arg: str_or_none(req_body.get(arg)) for arg in args} + logging.info(f"Parameters: {args}.") + return args_dict + + +def str_or_none(item: Any) -> str | None: + return str(item) if item is not None else None + + +def check_args(args: dict[str, str | None]) -> dict[str, str] | None: + if None in args.values(): + return None + else: + return {key: str(value) for key, value in args.items()} + + +def missing_parameters_repsonse() -> func.HttpResponse: + msg = "Required parameter not provided." + logging.critical(msg) + return func.HttpResponse( + msg, + status_code=400, + ) + + +def handle_response( + response: requests.Response, valid_codes: list[int], error_message: str +) -> func.HttpResponse | None: + logging.info(f"Response status code: {response.status_code}.") + logging.debug(f"Response contents: {response.text}.") + if response.status_code not in valid_codes: + return func.HttpResponse( + error_message, + status_code=400, + ) + else: + return None From f3c8838662af31188d947e37e194fad19b98ae8c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 2 Aug 2024 12:14:28 +0100 Subject: [PATCH 43/44] Update requirements --- .../resources/gitea_mirror/functions/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/resources/gitea_mirror/functions/requirements.txt b/data_safe_haven/resources/gitea_mirror/functions/requirements.txt index e69de29bb2..f2293605cf 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/requirements.txt +++ b/data_safe_haven/resources/gitea_mirror/functions/requirements.txt @@ -0,0 +1 @@ +requests From afdf574eef768a2957cbfe095c5b8deb4e889478 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 2 Aug 2024 12:21:16 +0100 Subject: [PATCH 44/44] Sort imports --- .../gitea_mirror/functions/create_mirror/__init__.py | 12 +++++++++--- .../gitea_mirror/functions/delete_mirror/__init__.py | 11 ++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py index 47e5b26f56..4125e488e1 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py +++ b/data_safe_haven/resources/gitea_mirror/functions/create_mirror/__init__.py @@ -3,10 +3,16 @@ import azure.functions as func import requests from requests.auth import HTTPBasicAuth - from shared_code import ( - get_args, check_args, missing_parameters_repsonse, timeout, gitea_host, api_root, - migrate_path, repos_path, handle_response + api_root, + check_args, + get_args, + gitea_host, + handle_response, + migrate_path, + missing_parameters_repsonse, + repos_path, + timeout, ) diff --git a/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py index c003015e04..a4aeb2f6b2 100644 --- a/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py +++ b/data_safe_haven/resources/gitea_mirror/functions/delete_mirror/__init__.py @@ -3,10 +3,15 @@ import azure.functions as func import requests from requests.auth import HTTPBasicAuth - from shared_code import ( - get_args, check_args, missing_parameters_repsonse, timeout, gitea_host, api_root, - repos_path, handle_response + api_root, + check_args, + get_args, + gitea_host, + handle_response, + missing_parameters_repsonse, + repos_path, + timeout, )