Skip to content

Commit

Permalink
record 1
Browse files Browse the repository at this point in the history
  • Loading branch information
@0xff-dev
0xff-dev committed Jan 29, 2024
1 parent 2715965 commit cd01b3e
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 101 deletions.
4 changes: 2 additions & 2 deletions controllers/evaluation/rag_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,13 +224,13 @@ func (r *RAGReconciler) phaseHandler(ctx context.Context, instance *evaluationar
}
return ctrl.Result{}, nil
case evaluationarcadiav1alpha1.JudgeLLMPhase:
err := r.JobGenerator(ctx, instance, curPhase, evaluationarcadiav1alpha1.UploadFilesPhase, evaluation.JudgeJob)
err := r.JobGenerator(ctx, instance, curPhase, evaluationarcadiav1alpha1.UploadFilesPhase, evaluation.JudgeJobGenerator(ctx, r.Client))
if err != nil && err != errJobNotDone {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
case evaluationarcadiav1alpha1.UploadFilesPhase:
err := r.JobGenerator(ctx, instance, curPhase, evaluationarcadiav1alpha1.CompletePhase, evaluation.UploadJob)
err := r.JobGenerator(ctx, instance, curPhase, evaluationarcadiav1alpha1.CompletePhase, evaluation.UploadJobGenerator(ctx, r.Client))
if err != nil && err != errJobNotDone {
return ctrl.Result{}, err
}
Expand Down
172 changes: 73 additions & 99 deletions pkg/evaluation/jobs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,14 @@ import (

"github.com/kubeagi/arcadia/api/base/v1alpha1"
evav1alpha1 "github.com/kubeagi/arcadia/api/evaluation/v1alpha1"
"github.com/kubeagi/arcadia/pkg/config"
"github.com/kubeagi/arcadia/pkg/llms"
)

const (
defaultPVCMountPath = "/data/evaluations"
defaultTestRagFile = "ragas.csv"
defaultMCImage = "kubeagi/minio-mc:RELEASE.2023-01-28T20-29-38Z"
)

func PhaseJobName(instance *evav1alpha1.RAG, phase evav1alpha1.RAGPhase) string {
Expand Down Expand Up @@ -125,6 +128,8 @@ func GenTestDataJob(instance *evav1alpha1.RAG) (*batchv1.Job, error) {
fmt.Sprintf("--application=%s", instance.Spec.Application.Name),
fmt.Sprintf("--input-dir=%s", defaultPVCMountPath),
"--output=csv",
"--merge=true",
fmt.Sprintf("--merge-file=%s", filepath.Join(defaultPVCMountPath, defaultTestRagFile)),
},
VolumeMounts: []v1.VolumeMount{
{
Expand Down Expand Up @@ -217,8 +222,9 @@ func JudgeJobGenerator(ctx context.Context, c client.Client) func(*evav1alpha1.R
ServiceAccountName: instance.Spec.ServiceAccountName,
Containers: []v1.Container{
{
Name: "judge-llm",
Image: "kubeagi/arcadia-eval",
Name: "judge-llm",
Image: "kubeagi/arcadia-eval",
WorkingDir: defaultPVCMountPath,
Command: []string{
"python3",
},
Expand Down Expand Up @@ -260,114 +266,82 @@ func JudgeJobGenerator(ctx context.Context, c client.Client) func(*evav1alpha1.R
return job, nil
}
}
func JudgeJob(instance *evav1alpha1.RAG) (*batchv1.Job, error) {
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Namespace: instance.Namespace,
Name: PhaseJobName(instance, evav1alpha1.JudgeLLMPhase),
Labels: map[string]string{
evav1alpha1.EvaluationJobLabels: instance.Name,

func UploadJobGenerator(ctx context.Context, client client.Client) func(*evav1alpha1.RAG) (*batchv1.Job, error) {
return func(instance *evav1alpha1.RAG) (*batchv1.Job, error) {
datasource, err := config.GetSystemDatasource(ctx, client, nil)
if err != nil {
return nil, err
}
url := datasource.Spec.Endpoint.URL
ns := datasource.Namespace
if datasource.Spec.Endpoint.AuthSecret.Namespace != nil {
ns = *datasource.Spec.Endpoint.AuthSecret.Namespace
}
data, err := datasource.Spec.Endpoint.AuthData(ctx, ns, client, nil)
if err != nil {
return nil, err
}

accessKeyID := string(data["rootUser"])
secretAccessKey := string(data["rootPassword"])

job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Namespace: instance.Namespace,
Name: PhaseJobName(instance, evav1alpha1.UploadFilesPhase),
Labels: map[string]string{
evav1alpha1.EvaluationJobLabels: instance.Name,
},
},
},
Spec: batchv1.JobSpec{
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
ServiceAccountName: instance.Spec.ServiceAccountName,
Containers: []v1.Container{
{
Name: "judge-llm",
Image: "perl:5.34.0",
Command: []string{
"perl",
},
Args: []string{
"-Mbignum=bpi",
"-wle",
"print bpi(2000)",
},
VolumeMounts: []v1.VolumeMount{
{
Name: "data",
MountPath: defaultPVCMountPath,
Spec: batchv1.JobSpec{
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
ServiceAccountName: instance.Spec.ServiceAccountName,
Containers: []v1.Container{
{
Name: "upload-result",
Image: defaultMCImage,
Command: []string{
"/bin/bash",
"-c",
fmt.Sprintf(`echo "upload result"
mc alias set oss $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY --insecure
mc --insecure cp -r %s/ oss/%s/evals/%s/%s`, defaultPVCMountPath, instance.Namespace, instance.Spec.Application.Name, instance.Name),
},
},
},
},
Volumes: []v1.Volume{
{
Name: "data",
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: instance.Name,
ReadOnly: false,
VolumeMounts: []v1.VolumeMount{
{
Name: "data",
MountPath: defaultPVCMountPath,
},
},
},
},
},
},
},
BackoffLimit: pointer.Int32(1),
Completions: pointer.Int32(1),
Parallelism: pointer.Int32(1),
Suspend: &instance.Spec.Suspend,
},
}
return job, nil
}

func UploadJob(instance *evav1alpha1.RAG) (*batchv1.Job, error) {
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
Namespace: instance.Namespace,
Name: PhaseJobName(instance, evav1alpha1.UploadFilesPhase),
Labels: map[string]string{
evav1alpha1.EvaluationJobLabels: instance.Name,
},
},
Spec: batchv1.JobSpec{
Template: v1.PodTemplateSpec{
Spec: v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever,
ServiceAccountName: instance.Spec.ServiceAccountName,
Containers: []v1.Container{
{
Name: "upload-result",
Image: "perl:5.34.0",
Command: []string{
"perl",
},
Args: []string{
"-Mbignum=bpi",
"-wle",
"print bpi(2000)",
},
VolumeMounts: []v1.VolumeMount{
{
Name: "data",
MountPath: defaultPVCMountPath,
Env: []v1.EnvVar{
{Name: "MINIO_ENDPOINT", Value: url},
{Name: "MINIO_ACCESS_KEY", Value: accessKeyID},
{Name: "MINIO_SECRET_KEY", Value: secretAccessKey},
},
},
},
},
Volumes: []v1.Volume{
{
Name: "data",
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: instance.Name,
ReadOnly: false,
Volumes: []v1.Volume{
{
Name: "data",
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: instance.Name,
ReadOnly: false,
},
},
},
},
},
},
BackoffLimit: pointer.Int32(1),
Completions: pointer.Int32(1),
Parallelism: pointer.Int32(1),
Suspend: &instance.Spec.Suspend,
},
BackoffLimit: pointer.Int32(1),
Completions: pointer.Int32(1),
Parallelism: pointer.Int32(1),
Suspend: &instance.Spec.Suspend,
},
}
return job, nil
}
return job, nil
}

0 comments on commit cd01b3e

Please sign in to comment.