From cd01b3ebb8f16c7a61cc6725ba4db268b70de624 Mon Sep 17 00:00:00 2001 From: 0xff-dev Date: Mon, 29 Jan 2024 17:26:30 +0800 Subject: [PATCH] record 1 --- controllers/evaluation/rag_controller.go | 4 +- pkg/evaluation/jobs.go | 172 ++++++++++------------- 2 files changed, 75 insertions(+), 101 deletions(-) diff --git a/controllers/evaluation/rag_controller.go b/controllers/evaluation/rag_controller.go index dc022731c..fc7c54f03 100644 --- a/controllers/evaluation/rag_controller.go +++ b/controllers/evaluation/rag_controller.go @@ -224,13 +224,13 @@ func (r *RAGReconciler) phaseHandler(ctx context.Context, instance *evaluationar } return ctrl.Result{}, nil case evaluationarcadiav1alpha1.JudgeLLMPhase: - err := r.JobGenerator(ctx, instance, curPhase, evaluationarcadiav1alpha1.UploadFilesPhase, evaluation.JudgeJob) + err := r.JobGenerator(ctx, instance, curPhase, evaluationarcadiav1alpha1.UploadFilesPhase, evaluation.JudgeJobGenerator(ctx, r.Client)) if err != nil && err != errJobNotDone { return ctrl.Result{}, err } return ctrl.Result{}, nil case evaluationarcadiav1alpha1.UploadFilesPhase: - err := r.JobGenerator(ctx, instance, curPhase, evaluationarcadiav1alpha1.CompletePhase, evaluation.UploadJob) + err := r.JobGenerator(ctx, instance, curPhase, evaluationarcadiav1alpha1.CompletePhase, evaluation.UploadJobGenerator(ctx, r.Client)) if err != nil && err != errJobNotDone { return ctrl.Result{}, err } diff --git a/pkg/evaluation/jobs.go b/pkg/evaluation/jobs.go index 4b9255168..bc84e3b5d 100644 --- a/pkg/evaluation/jobs.go +++ b/pkg/evaluation/jobs.go @@ -29,11 +29,14 @@ import ( "github.com/kubeagi/arcadia/api/base/v1alpha1" evav1alpha1 "github.com/kubeagi/arcadia/api/evaluation/v1alpha1" + "github.com/kubeagi/arcadia/pkg/config" "github.com/kubeagi/arcadia/pkg/llms" ) const ( defaultPVCMountPath = "/data/evaluations" + defaultTestRagFile = "ragas.csv" + defaultMCImage = "kubeagi/minio-mc:RELEASE.2023-01-28T20-29-38Z" ) func PhaseJobName(instance *evav1alpha1.RAG, phase evav1alpha1.RAGPhase) string { @@ -125,6 +128,8 @@ func GenTestDataJob(instance *evav1alpha1.RAG) (*batchv1.Job, error) { fmt.Sprintf("--application=%s", instance.Spec.Application.Name), fmt.Sprintf("--input-dir=%s", defaultPVCMountPath), "--output=csv", + "--merge=true", + fmt.Sprintf("--merge-file=%s", filepath.Join(defaultPVCMountPath, defaultTestRagFile)), }, VolumeMounts: []v1.VolumeMount{ { @@ -217,8 +222,9 @@ func JudgeJobGenerator(ctx context.Context, c client.Client) func(*evav1alpha1.R ServiceAccountName: instance.Spec.ServiceAccountName, Containers: []v1.Container{ { - Name: "judge-llm", - Image: "kubeagi/arcadia-eval", + Name: "judge-llm", + Image: "kubeagi/arcadia-eval", + WorkingDir: defaultPVCMountPath, Command: []string{ "python3", }, @@ -260,114 +266,82 @@ func JudgeJobGenerator(ctx context.Context, c client.Client) func(*evav1alpha1.R return job, nil } } -func JudgeJob(instance *evav1alpha1.RAG) (*batchv1.Job, error) { - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: instance.Namespace, - Name: PhaseJobName(instance, evav1alpha1.JudgeLLMPhase), - Labels: map[string]string{ - evav1alpha1.EvaluationJobLabels: instance.Name, + +func UploadJobGenerator(ctx context.Context, client client.Client) func(*evav1alpha1.RAG) (*batchv1.Job, error) { + return func(instance *evav1alpha1.RAG) (*batchv1.Job, error) { + datasource, err := config.GetSystemDatasource(ctx, client, nil) + if err != nil { + return nil, err + } + url := datasource.Spec.Endpoint.URL + ns := datasource.Namespace + if datasource.Spec.Endpoint.AuthSecret.Namespace != nil { + ns = *datasource.Spec.Endpoint.AuthSecret.Namespace + } + data, err := datasource.Spec.Endpoint.AuthData(ctx, ns, client, nil) + if err != nil { + return nil, err + } + + accessKeyID := string(data["rootUser"]) + secretAccessKey := string(data["rootPassword"]) + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: instance.Namespace, + Name: PhaseJobName(instance, evav1alpha1.UploadFilesPhase), + Labels: map[string]string{ + evav1alpha1.EvaluationJobLabels: instance.Name, + }, }, - }, - Spec: batchv1.JobSpec{ - Template: v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyNever, - ServiceAccountName: instance.Spec.ServiceAccountName, - Containers: []v1.Container{ - { - Name: "judge-llm", - Image: "perl:5.34.0", - Command: []string{ - "perl", - }, - Args: []string{ - "-Mbignum=bpi", - "-wle", - "print bpi(2000)", - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "data", - MountPath: defaultPVCMountPath, + Spec: batchv1.JobSpec{ + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: instance.Spec.ServiceAccountName, + Containers: []v1.Container{ + { + Name: "upload-result", + Image: defaultMCImage, + Command: []string{ + "/bin/bash", + "-c", + fmt.Sprintf(`echo "upload result" +mc alias set oss $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY --insecure +mc --insecure cp -r %s/ oss/%s/evals/%s/%s`, defaultPVCMountPath, instance.Namespace, instance.Spec.Application.Name, instance.Name), }, - }, - }, - }, - Volumes: []v1.Volume{ - { - Name: "data", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: instance.Name, - ReadOnly: false, + VolumeMounts: []v1.VolumeMount{ + { + Name: "data", + MountPath: defaultPVCMountPath, + }, }, - }, - }, - }, - }, - }, - BackoffLimit: pointer.Int32(1), - Completions: pointer.Int32(1), - Parallelism: pointer.Int32(1), - Suspend: &instance.Spec.Suspend, - }, - } - return job, nil -} - -func UploadJob(instance *evav1alpha1.RAG) (*batchv1.Job, error) { - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: instance.Namespace, - Name: PhaseJobName(instance, evav1alpha1.UploadFilesPhase), - Labels: map[string]string{ - evav1alpha1.EvaluationJobLabels: instance.Name, - }, - }, - Spec: batchv1.JobSpec{ - Template: v1.PodTemplateSpec{ - Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyNever, - ServiceAccountName: instance.Spec.ServiceAccountName, - Containers: []v1.Container{ - { - Name: "upload-result", - Image: "perl:5.34.0", - Command: []string{ - "perl", - }, - Args: []string{ - "-Mbignum=bpi", - "-wle", - "print bpi(2000)", - }, - VolumeMounts: []v1.VolumeMount{ - { - Name: "data", - MountPath: defaultPVCMountPath, + Env: []v1.EnvVar{ + {Name: "MINIO_ENDPOINT", Value: url}, + {Name: "MINIO_ACCESS_KEY", Value: accessKeyID}, + {Name: "MINIO_SECRET_KEY", Value: secretAccessKey}, }, }, }, - }, - Volumes: []v1.Volume{ - { - Name: "data", - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: instance.Name, - ReadOnly: false, + Volumes: []v1.Volume{ + { + Name: "data", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: instance.Name, + ReadOnly: false, + }, }, }, }, }, }, + BackoffLimit: pointer.Int32(1), + Completions: pointer.Int32(1), + Parallelism: pointer.Int32(1), + Suspend: &instance.Spec.Suspend, }, - BackoffLimit: pointer.Int32(1), - Completions: pointer.Int32(1), - Parallelism: pointer.Int32(1), - Suspend: &instance.Spec.Suspend, - }, + } + return job, nil } - return job, nil }