From 14bd84780b0f0ee819009a3fa4381cc240a33625 Mon Sep 17 00:00:00 2001 From: philmcmahon Date: Fri, 26 Jan 2024 16:51:52 +0000 Subject: [PATCH] Tidy up cdk --- package-lock.json | 68 ++++++------- packages/cdk/lib/transcription-service.ts | 114 ++++++++++++++++++++-- 2 files changed, 138 insertions(+), 44 deletions(-) diff --git a/package-lock.json b/package-lock.json index be7461d1..b40bac9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3860,13 +3860,13 @@ }, "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { "version": "1.0.2", - "dev": true, + "extraneous": true, "inBundle": true, "license": "Apache-2.0" }, "node_modules/aws-cdk-lib/node_modules/ajv": { "version": "8.12.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3882,7 +3882,7 @@ }, "node_modules/aws-cdk-lib/node_modules/ansi-regex": { "version": "5.0.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -3891,7 +3891,7 @@ }, "node_modules/aws-cdk-lib/node_modules/ansi-styles": { "version": "4.3.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3906,7 +3906,7 @@ }, "node_modules/aws-cdk-lib/node_modules/astral-regex": { "version": "2.0.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -3915,13 +3915,13 @@ }, "node_modules/aws-cdk-lib/node_modules/balanced-match": { "version": "1.0.2", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/brace-expansion": { "version": "1.1.11", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3931,7 +3931,7 @@ }, "node_modules/aws-cdk-lib/node_modules/case": { "version": "1.6.3", - "dev": true, + "extraneous": true, "inBundle": true, "license": "(MIT OR GPL-3.0-or-later)", "engines": { @@ -3940,7 +3940,7 @@ }, "node_modules/aws-cdk-lib/node_modules/color-convert": { "version": "2.0.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3952,31 +3952,31 @@ }, "node_modules/aws-cdk-lib/node_modules/color-name": { "version": "1.1.4", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/concat-map": { "version": "0.0.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/emoji-regex": { "version": "8.0.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { "version": "3.1.3", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/fs-extra": { "version": "11.2.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -3990,13 +3990,13 @@ }, "node_modules/aws-cdk-lib/node_modules/graceful-fs": { "version": "4.2.11", - "dev": true, + "extraneous": true, "inBundle": true, "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/ignore": { "version": "5.3.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -4005,7 +4005,7 @@ }, "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { "version": "3.0.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -4014,13 +4014,13 @@ }, "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { "version": "1.0.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/jsonfile": { "version": "6.1.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -4032,7 +4032,7 @@ }, "node_modules/aws-cdk-lib/node_modules/jsonschema": { "version": "1.4.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -4041,13 +4041,13 @@ }, "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { "version": "4.4.2", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT" }, "node_modules/aws-cdk-lib/node_modules/lru-cache": { "version": "6.0.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "ISC", "dependencies": { @@ -4059,7 +4059,7 @@ }, "node_modules/aws-cdk-lib/node_modules/minimatch": { "version": "3.1.2", - "dev": true, + "extraneous": true, "inBundle": true, "license": "ISC", "dependencies": { @@ -4071,7 +4071,7 @@ }, "node_modules/aws-cdk-lib/node_modules/punycode": { "version": "2.3.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -4080,7 +4080,7 @@ }, "node_modules/aws-cdk-lib/node_modules/require-from-string": { "version": "2.0.2", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -4089,7 +4089,7 @@ }, "node_modules/aws-cdk-lib/node_modules/semver": { "version": "7.5.4", - "dev": true, + "extraneous": true, "inBundle": true, "license": "ISC", "dependencies": { @@ -4104,7 +4104,7 @@ }, "node_modules/aws-cdk-lib/node_modules/slice-ansi": { "version": "4.0.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -4121,7 +4121,7 @@ }, "node_modules/aws-cdk-lib/node_modules/string-width": { "version": "4.2.3", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -4135,7 +4135,7 @@ }, "node_modules/aws-cdk-lib/node_modules/strip-ansi": { "version": "6.0.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "dependencies": { @@ -4147,7 +4147,7 @@ }, "node_modules/aws-cdk-lib/node_modules/table": { "version": "6.8.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "BSD-3-Clause", "dependencies": { @@ -4163,7 +4163,7 @@ }, "node_modules/aws-cdk-lib/node_modules/universalify": { "version": "2.0.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "MIT", "engines": { @@ -4172,7 +4172,7 @@ }, "node_modules/aws-cdk-lib/node_modules/uri-js": { "version": "4.4.1", - "dev": true, + "extraneous": true, "inBundle": true, "license": "BSD-2-Clause", "dependencies": { @@ -4181,13 +4181,13 @@ }, "node_modules/aws-cdk-lib/node_modules/yallist": { "version": "4.0.0", - "dev": true, + "extraneous": true, "inBundle": true, "license": "ISC" }, "node_modules/aws-cdk-lib/node_modules/yaml": { "version": "1.10.2", - "dev": true, + "extraneous": true, "inBundle": true, "license": "ISC", "engines": { diff --git a/packages/cdk/lib/transcription-service.ts b/packages/cdk/lib/transcription-service.ts index 981f6d1d..6228db29 100644 --- a/packages/cdk/lib/transcription-service.ts +++ b/packages/cdk/lib/transcription-service.ts @@ -1,13 +1,24 @@ -import { GuApiLambda } from '@guardian/cdk'; -import { GuCertificate } from '@guardian/cdk/lib/constructs/acm'; -import { GuStack } from '@guardian/cdk/lib/constructs/core'; -import type { GuStackProps } from '@guardian/cdk/lib/constructs/core'; -import { GuCname } from '@guardian/cdk/lib/constructs/dns'; -import { GuardianAwsAccounts } from '@guardian/private-infrastructure-config'; -import { type App, Duration } from 'aws-cdk-lib'; -import { EndpointType } from 'aws-cdk-lib/aws-apigateway'; -import { Effect, PolicyStatement } from 'aws-cdk-lib/aws-iam'; -import { Runtime } from 'aws-cdk-lib/aws-lambda'; +import {GuApiLambda} from '@guardian/cdk'; +import {GuCertificate} from '@guardian/cdk/lib/constructs/acm'; +import type {GuStackProps} from '@guardian/cdk/lib/constructs/core'; +import {GuAmiParameter, GuDistributionBucketParameter, GuStack} from '@guardian/cdk/lib/constructs/core'; +import {GuCname} from '@guardian/cdk/lib/constructs/dns'; +import {GuVpc, SubnetType} from "@guardian/cdk/lib/constructs/ec2"; +import {GuardianAwsAccounts} from '@guardian/private-infrastructure-config'; +import {type App, Duration} from 'aws-cdk-lib'; +import {EndpointType} from 'aws-cdk-lib/aws-apigateway'; +import {AutoScalingGroup, BlockDeviceVolume, SpotAllocationStrategy} from "aws-cdk-lib/aws-autoscaling"; +import { + InstanceClass, + InstanceSize, + InstanceType, + LaunchTemplate, + MachineImage, + SpotInstanceInterruption, + UserData +} from "aws-cdk-lib/aws-ec2"; +import {Effect, PolicyStatement} from 'aws-cdk-lib/aws-iam'; +import {Runtime} from 'aws-cdk-lib/aws-lambda'; export class TranscriptionService extends GuStack { constructor(scope: App, id: string, props: GuStackProps) { @@ -15,8 +26,14 @@ export class TranscriptionService extends GuStack { const APP_NAME = 'transcription-service'; const apiId = `${APP_NAME}-${props.stage}`; + const isProd = props.stage === 'PROD'; if (!props.env?.region) throw new Error('region not provided in props'); + const workerAmi = new GuAmiParameter(this, { + app: `${APP_NAME}-worker`, + description: "AMI to use for the worker instances" + }) + const ssmPrefix = `arn:aws:ssm:${props.env.region}:${GuardianAwsAccounts.Investigations}:parameter`; const ssmPath = `${this.stage}/${this.stack}/${APP_NAME}`; const domainName = @@ -67,5 +84,82 @@ export class TranscriptionService extends GuStack { ttl: Duration.hours(1), resourceRecord: apiDomain.domainNameAliasDomainName, }); + + // worker + + const workerApp = `${APP_NAME}-worker` + const userData = UserData.forLinux({ shebang: "#!/bin/bash" + }) + // basic placeholder commands + userData.addCommands([ + `aws s3 cp s3://${GuDistributionBucketParameter.getInstance(this).valueAsString}/${props.stack}/${props.stage}/${APP_NAME}/worker.zip`, + `unzip worker.zip`, + `node index.js` + ].join("\n")) + + const launchTemplate = new LaunchTemplate(this, "TranscriptionWorkerLaunchTemplate", { + machineImage: MachineImage.genericLinux({"eu-west-1": workerAmi.valueAsString}), + spotOptions: { + interruptionBehavior: SpotInstanceInterruption.TERMINATE, + maxPrice: 0.6202, // the on-demand price of a c7g.4xlarge in eu-west-1 + }, + instanceType: InstanceType.of(InstanceClass.C7G, InstanceSize.XLARGE4), + // the size of this block device will determine the max input file size for transcription. In future we could + // attach the block device on startup once we know how large the file to be transcribed is, or try some kind + // of streaming approach to the transcription so we don't need the whole file on disk + blockDevices: [ + { + deviceName: "/dev/sda1", + // assuming that we intend to support video files, 50GB seems a reasonable starting point (beyond that + // the browser upload might struggle anyway) + volume: BlockDeviceVolume.ebs(50) + } + ], + userData + }) + + // instance types we are happy to use for workers. Note - order matters as when launching 'on demand' instances + // the ASG will start at the top of the list and work down until it manages to launch an instance + const acceptableInstanceTypes = [ + InstanceType.of(InstanceClass.C7G, InstanceSize.XLARGE4), + InstanceType.of(InstanceClass.C6G, InstanceSize.XLARGE4), + InstanceType.of(InstanceClass.M7G, InstanceSize.XLARGE4), + InstanceType.of(InstanceClass.C7G, InstanceSize.XLARGE8), + InstanceType.of(InstanceClass.C6G, InstanceSize.XLARGE8) + ] + + // worker autoscaling group + // unfortunately GuAutoscalingGroup doesn't support having a mixedInstancesPolicy so using the basic ASG here + new AutoScalingGroup(this, "TransciptionWorkerASG", { + minCapacity: 0, + maxCapacity: isProd ? 20 : 4, + autoScalingGroupName: `transcription-service-workers-${this.stage}`, + vpc: GuVpc.fromIdParameter(this, "InvestigationsInternetEnabledVpc", { + availabilityZones: ["eu-west-1a", "eu-west-1b", "eu-west-1c"], + }), + vpcSubnets: { + subnets: GuVpc.subnetsFromParameter(this, { + type: SubnetType.PRIVATE, + app: workerApp + }) + }, + // instances should shut themselves down when they have finished the transcription - they should not be terminated + // by the ASG + newInstancesProtectedFromScaleIn: true, + mixedInstancesPolicy: { + launchTemplate, + instancesDistribution: { + // 0 is the default, including this here just to make it more obvious what's happening + onDemandBaseCapacity: 0, + // if this value is set to 100, then we won't use spot instances at all, if it is 0 then we use 100% spot + onDemandPercentageAboveBaseCapacity: 100, + spotAllocationStrategy: SpotAllocationStrategy.CAPACITY_OPTIMIZED, + }, + launchTemplateOverrides: acceptableInstanceTypes.map(instanceType => ({ + instanceType + })) + } + }) + } }