Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Add mirror creation script #2008

Draft
wants to merge 46 commits into
base: develop-v5.1.0
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 32 commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
bc9c5ee
WIP: Add mirror creation script
JimMadge Jul 10, 2024
c41196a
Add description
JimMadge Jul 10, 2024
d949996
Add delete script
JimMadge Jul 10, 2024
9405f9a
Correct pull mirror creation
JimMadge Jul 10, 2024
9a26635
Configure pull mirror repository
JimMadge Jul 10, 2024
e8b2c17
Fix linting
JimMadge Jul 10, 2024
9e1a5f2
Remove unused variables
JimMadge Jul 10, 2024
7f8fe87
Rewrite mirror creation as function app
JimMadge Jul 10, 2024
fabd2b2
Restructure directory
JimMadge Jul 10, 2024
68e85f2
Rename function and api root
JimMadge Jul 10, 2024
3b15ff2
Remove unused method to get args
JimMadge Jul 10, 2024
783999e
Correct status code check
JimMadge Jul 10, 2024
5ad78a4
Add exception for magic number
JimMadge Jul 10, 2024
bef8bcc
Add typings for azure functions
JimMadge Jul 10, 2024
00f3036
Remove empty comment
JimMadge Jul 10, 2024
f453d39
Remove old script
JimMadge Jul 11, 2024
428c649
Put both routes in one function app
JimMadge Jul 11, 2024
2e2dae3
Generalise argument parsing
JimMadge Jul 11, 2024
eb7bdab
Generalise response handling
JimMadge Jul 11, 2024
99dc8cb
Add return statement
JimMadge Jul 11, 2024
0a41555
Add basic test
JimMadge Jul 11, 2024
fb41278
Add delete mirror test
JimMadge Jul 11, 2024
003d24f
Add test for missing arguments
JimMadge Jul 11, 2024
8b63b8e
Add basic test for create mirror
JimMadge Jul 11, 2024
b88a380
Add test for failure to delete mirror
JimMadge Jul 11, 2024
76e5f44
Add failure tests for mirror creation
JimMadge Jul 11, 2024
edd65dc
Remove some magic strings
JimMadge Jul 11, 2024
e9c8f29
Run lint:fmt
JimMadge Jul 11, 2024
9cf8350
WIP: Add app service infrastructure
JimMadge Jul 12, 2024
6863e17
Merge remote-tracking branch 'origin/develop' into gitea_mirror_creation
JimMadge Jul 30, 2024
a91f17f
WIP remove resource group
JimMadge Jul 30, 2024
e8be73a
WIP: Add webapp
JimMadge Jul 30, 2024
b5cc5e1
Fix linting
JimMadge Jul 30, 2024
fd82175
Add apps component to sre
JimMadge Jul 30, 2024
0b6ba29
Fix import
JimMadge Jul 30, 2024
9031e8b
Merge remote-tracking branch 'origin/develop' into gitea_mirror_creation
JimMadge Jul 30, 2024
491a27a
Correct tags and HTTPS enum
JimMadge Jul 31, 2024
0753fa1
Fix arguments
JimMadge Jul 31, 2024
7fdc5fb
Change Function App name
JimMadge Jul 31, 2024
8dc0356
Update WebApp type
JimMadge Jul 31, 2024
e495c66
WIP: Add connection string
JimMadge Aug 1, 2024
6dffae9
Set webapp to always on
JimMadge Aug 1, 2024
5aec4e5
Set Python version
JimMadge Aug 1, 2024
97e4ca8
Restructure using v1 programming model
JimMadge Aug 1, 2024
f3c8838
Update requirements
JimMadge Aug 2, 2024
afdf574
Sort imports
JimMadge Aug 2, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
181 changes: 181 additions & 0 deletions data_safe_haven/infrastructure/programs/sre/apps.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
"""Pulumi component for SRE function/web apps"""

from collections.abc import Mapping

from pulumi import ComponentResource, FileArchive, Input, Output, ResourceOptions
from pulumi_azure_native import (
resources,
storage,
web,
)

from data_safe_haven.functions import (
alphanumeric,
truncate_tokens,
)
from data_safe_haven.infrastructure.common import (
get_name_from_rg,
)
from data_safe_haven.resources import resources_path


class SREAppsProps:
"""Properties for SREAppsComponent"""

def __init__(
self,
location: Input[str],
resource_group: Input[resources.ResourceGroup],
):
self.location = location
self.resource_group_name = Output.from_input(resource_group).apply(
get_name_from_rg
)


class SREAppsComponent(ComponentResource):
"""Deploy SRE function/web apps with Pulumi"""

def __init__(
self,
name: str,
stack_name: str,
props: SREAppsProps,
opts: ResourceOptions | None = None,
tags: Input[Mapping[str, Input[str]]] | None = None,
) -> None:
super().__init__("dsh:sre:AppsComponent", name, {}, opts)
child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self))
child_tags = tags if tags else {}

# Deploy storage account
# The storage account holds app data/configuration
storage_account = storage.StorageAccount(
f"{self._name}_storage_account",
account_name=alphanumeric(
f"{''.join(truncate_tokens(stack_name.split('-'), 14))}apps"
)[:24],
kind=storage.Kind.STORAGE_V2,
location=props.location,
resource_group_name=props.resource_group_name,
sku=storage.SkuArgs(name=storage.SkuName.STANDARD_GRS),
opts=child_opts,
tags=child_tags,
)

# Create function apps container
container = storage.BlobContainer(
f"{self._name}_container_functions",
account_name=storage_account.name,
container_name="functions",
public_access=storage.PublicAccess.NONE,
resource_group_name=props.resource_group_name,
opts=ResourceOptions.merge(
child_opts,
ResourceOptions(parent=storage_account),
),
tags=child_tags,
)

# Upload Gitea mirror function app
blob_gitea_mirror = storage.Blob(
f"{self._name}_blob_gitea_mirror",
account_name=storage_account.name,
container_name=container.name,
resource_group_name=props.resource_group_name,
source=FileArchive(
str((resources_path / "gitea_mirror" / "functions").absolute()),
),
opts=ResourceOptions.merge(
child_opts,
ResourceOptions(parent=container),
),
tags=child_tags,
)

# Get URL of app blob
blob_url = get_blob_url(
blob=blob_gitea_mirror,
container=container,
storage_account=storage_account,
resource_group_name=props.resource_group_name,
)

# Deploy service plan
app_service_plan = web.AppServicePlan(
f"{self._name}_app_service_plan",
kind="linux",
location=props.location,
name=f"{stack_name}-app-service-plan",
resource_group_name=props.resource_group_name,
sku={
"name": "B1",
"tier": "Basic",
"size": "B1",
"family": "B",
"capacity": 1,
},
)

# Deploy app
web.WebApp(
f"{self._name}_web_app",
enabled=True,
https_only=True,
kind="FunctionApp",
location=props.location,
name="giteamirror",
resource_group_name=props.resource_group_name,
server_farm_id=app_service_plan.id,
site_config=web.SiteConfig(
app_settings=[
{"name": "runtime", "value": "python"},
{"name": "FUNCTIONS_WORKER_RUNTIME", "value": "python"},
{"name": "WEBSITE_RUN_FROM_PACKAGE", "value": blob_url},
{"name": "FUNCTIONS_EXTENSION_VERSION", "value": "~4"},
],
)
)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason we prefer the gitea mirror to run as a webapp rather than an ACI? Is it cheaper/easier/better in some other way to do this?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This part isn't the Gitea instance, it is a webapp that uses the Gitea API to create/delete mirrors.

I am hoping that using the Azure Functions framework we can avoid the complication of managing flask/fastapi/etc. and use Entra ID authentication.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unless there is a reason not to, I think we should deploy all Gitea instances the same way.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking that that now, there might be a fair bit of work to either generalise the Gitea configuration or make a new Pulumi ComponentResource.

I think we will want the mirror instance to be fairly different,

  • No login
  • Browseable without login
  • No registration
  • No LDAP authentication



def get_blob_url(
blob: Input[storage.Blob],
container: Input[storage.BlobContainer],
storage_account: Input[storage.StorageAccount],
resource_group_name: Input[str],
) -> Output[str]:
sas = storage.list_storage_account_service_sas_output(
account_name=storage_account.name,
protocols=storage.HttpProtocol.Https,
# shared_access_expiry_time="2030-01-01",
# shared_access_start_time="2021-01-01",
resource_group_name=resource_group_name,
# Access to container
resource=storage.SignedResource.C,
# Read access
permissions=storage.Permissions.R,
canonicalized_resource=Output.format(
"/blob/{account_name}/{container_name}",
account_name=storage_account.name,
container_name=container.name,
),
content_type="application/json",
cache_control="max-age=5",
content_disposition="inline",
content_encoding="deflate",
)
token = sas.service_sas_token
# return Output.format(
# "https://{0}.blob.core.windows.net/{1}/{2}?{3}",
# storage_account.name,
# container.name,
# blob.name,
# token,
# )
return Output.format(
"https://{storage_account_name}.blob.core.windows.net/{container_name}/{blob_name}?{token}",
storage_account_name=storage_account.name,
container_name=container.name,
blob_name=blob.name,
token=token,
)
48 changes: 48 additions & 0 deletions data_safe_haven/resources/gitea_mirror/functions/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
bin
obj
csx
.vs
edge
Publish

*.user
*.suo
*.cscfg
*.Cache
project.lock.json

/packages
/TestResults

/tools/NuGet.exe
/App_Data
/secrets
/data
.secrets
appsettings.json
local.settings.json

node_modules
dist

# Local python packages
.python_packages/

# Python Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# Azurite artifacts
__blobstorage__
__queuestorage__
__azurite_db*__.json
Loading
Loading