Single binary instructions

kcdmx2024/README.md

@@ -0,0 +1,89 @@
+# KCD Mexico workshop: building a cloud-native platform for ML training from scratch
+
+This is the logical architecture you'll be deploying and using:
+
+![](./img/logicalarch.png)
+
+## Pre-requisites
+
+1. Docker Desktop
+2. kubectl
+3. minikube
+4. python 3.9+
+
+## Preparing a Kubernetes environment
+
+1. Launch a new cluster
+
+```bash
+minikube start --driver=docker
+```
+
+2. Download the manifest for the Flyte dependencies:
+```bash
+curl -sl https://raw.githubusercontent.com/flyte-conference-talks/kcdmexico-2024/manifests/flyte-resources.yaml > flyte-resources.yaml
+```
+3. Submit the manifest:
+```bash
+kubectl create -f flyte-resources.yaml
+```
+3. Install Flyte:
+```bash
+helm install flyte-binary flyteorg/flyte-binary  --values values.yaml -n flyte
+```
+4. In three separate Terminal windows, start port-forwarding sessions to the following components:
+
+Web console
+```
+kubectl -n flyte port-forward service/flyte-binary-http 8088:8088 
+```
+
+API endpoint
+```
+kubectl -n flyte port-forward service/flyte-binary-grpc 8089:8089 
+```
+minio (blob storage)
+```
+kubectl -n flyte port-forward service/minio 9000:9000 
+```
+
+5. Edit the `$HOME/.flyte/config.yaml` file to reflect the following:
+```yaml
+admin:
+  endpoint: localhost:8089
+  insecure: true
+  authType: Pkce
+```
+6. Add and entry to your local DNS file so your `pyflyte` client is able to resolve the `minio` service name:
+```bash
+sudo vi /etc hosts
+
+# Host Database
+#
+# localhost is used to configure the loopback interface
+# when the system is booting.  Do not change this entry.
+##
+127.0.0.1       minio.flyte.svc.cluster.local
+```
+
+7. Download this demo workflow or simply start developing your own:
+
+``` bash
+curl -sl https://raw.githubusercontent.com/davidmirror-ops/flyte-the-hard-way/main/docs/on-premises/microk8s/demo.py > demo.py
+```
+
+8. Submit the workflow:
+``` bash
+pyflyte run --remote demo.py wf
+```
+Example output:
+``` bash
+Go to https://localhost:8089/console/projects/flytesnacks/domains/development/executions/f63a3e948256f4fd1b81 to see execution in the console.
+```
+> NOTE: instead of `8089` use `8088` as the port you connect to in the browser with the above link. All these limitations don't exist in production environments that make use of an Ingress or Service Mesh.
+
+Follow the link and observe your workflow succeeding!
+> The first run will take some time due to the download of the flytekit docker image
+
+### Congratulations
+You have setup a working Flyte environment on minikube

kcdmx2024/manifests/flyte-resources.yaml

@@ -0,0 +1,178 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: flyte
+  labels:
+    name: flyte
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: flyte-binary-inline-config-secret
+  namespace: flyte
+type: Opaque
+stringData:
+  202-database-secrets.yaml: |
+    database:
+      postgres:
+        password: "postgres"
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: postgresql-pvc
+  namespace: flyte
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: minio-pvc
+  namespace: flyte
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 20Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres
+  namespace: flyte
+  labels: 
+    app.kubernetes.io/name: postgres
+spec:
+  type: NodePort
+  ports:
+  - name: postgres
+    port: 5432
+    nodePort: 30089
+    protocol: TCP
+    targetPort: postgres
+  selector: 
+    app.kubernetes.io/name: postgres
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: postgres
+  namespace: flyte
+  labels: 
+    app.kubernetes.io/name: postgres
+spec:
+  replicas: 1
+  selector:
+    matchLabels: 
+      app.kubernetes.io/name: postgres
+  template:
+    metadata:
+      labels: 
+        app.kubernetes.io/name: postgres
+    spec:
+      containers:
+      - image: "ecr.flyte.org/ubuntu/postgres:13-21.04_beta"
+        imagePullPolicy: "IfNotPresent"
+        name: postgres
+        env:
+        - name: POSTGRES_PASSWORD
+          value: postgres #set your own
+        - name: POSTGRES_USER
+          value: postgres
+        - name: POSTGRES_DB
+          value: flyte
+        ports:
+        - containerPort: 5432
+          name: postgres
+        resources: 
+          limits:
+            cpu: 1000m
+            memory: 512Mi
+          requests:
+            cpu: 10m
+            memory: 128Mi
+        volumeMounts:
+        - name: postgres-storage
+          mountPath: /var/lib/postgresql/data
+      volumes:
+        - name: postgres-storage
+          persistentVolumeClaim:
+            claimName: postgresql-pvc
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+  namespace: flyte
+  labels: 
+    app.kubernetes.io/name: minio
+spec:
+  replicas: 1
+  selector:
+    matchLabels: 
+      app.kubernetes.io/name: minio
+  template:
+    metadata:
+      labels: 
+        app.kubernetes.io/name: minio
+    spec:
+      containers:
+      - image: "bitnami/minio:2023.7.18-debian-11-r2"
+        imagePullPolicy: "IfNotPresent"
+        name: minio
+        env:
+          #change the following values if needed. Make sure to apply these changes also to the corresponding fields in the local-values.yaml file
+          - name: MINIO_ROOT_USER
+            value: minio 
+          - name: MINIO_ROOT_PASSWORD
+            value: miniostorage
+          - name: MINIO_DEFAULT_BUCKETS
+            value: flyte
+        ports:
+        - containerPort: 9000
+          name: minio
+        - containerPort: 9001
+          name: minio-console
+        resources: 
+          limits:
+            cpu: 200m
+            memory: 512Mi
+          requests:
+            cpu: 10m
+            memory: 128Mi
+        volumeMounts:
+        - name: minio-storage
+          mountPath: /data
+      volumes:
+      - name: minio-storage
+        persistentVolumeClaim:
+          claimName: minio-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: flyte
+  labels: 
+    app.kubernetes.io/name: minio
+spec:
+  type: NodePort
+  ports:
+  - name: minio
+    nodePort: 30084
+    port: 9000
+    protocol: TCP
+    targetPort: minio
+  - name: minio-console
+    nodePort: 30088
+    port: 9001
+    protocol: TCP
+    targetPort: minio-console
+  selector: 
+    app.kubernetes.io/name: minio

kcdmx2024/values.yaml

@@ -0,0 +1,39 @@
+configuration:
+  database:
+    username: postgres
+    host: postgres.flyte.svc.cluster.local
+    dbname: flyte 
+  storage:
+    type: minio
+    metadataContainer: flyte #This is the default bucket created with Minio. Controlled by the MINIO_DEFAULT_BUCKETS env var in the local-flyte-resources.yaml manifest
+    userDataContainer: flyte
+    provider: s3
+    providerConfig:
+      s3:
+        authType: "accesskey"
+        endpoint: "http://minio.flyte.svc.cluster.local:57149"
+        accessKey: "minio"
+        secretKey: "miniostorage" #If you need to change this parameters, refer to the local-flyte-resources.yaml manifest and adjust the MINIO_ROOT_PASSWORD env var
+        disableSSL: "true"
+        secure: "false"
+        v2Signing: "true"
+
+  inlineSecretRef: flyte-binary-inline-config-secret  
+  inline:
+    plugins:
+      k8s:
+        inject-finalizer: true
+        default-env-vars:
+          - FLYTE_AWS_ENDPOINT: "http://minio.flyte.svc.cluster.local:9000"
+          - FLYTE_AWS_ACCESS_KEY_ID: "minio"
+          - FLYTE_AWS_SECRET_ACCESS_KEY: "miniostorage" #Use the same value as the MINIO_ROOT_PASSWORD
+
+    task_resources:
+      defaults: 
+        cpu: 100m
+        memory: 500Mi #change default requested resources and limits to fit your needs
+      limits:
+        memory: 2Gi
+
+serviceAccount:
+  create: true