feat:the time consumed for data processing.

kubeagi · Mar 22, 2024 · 80b85cb · 80b85cb
1 parent e0138f3
commit 80b85cb
Show file tree

Hide file tree

Showing 8 changed files with 583 additions and 2 deletions.
diff --git a/apiserver/graph/generated/generated.go b/apiserver/graph/generated/generated.go
diff --git a/apiserver/graph/generated/models_gen.go b/apiserver/graph/generated/models_gen.go
diff --git a/apiserver/graph/schema/dataprocessing.gql b/apiserver/graph/schema/dataprocessing.gql
@@ -11,6 +11,7 @@ query allDataProcessListByPage($input: AllDataProcessListByPageInput!){
         post_data_set_name
         post_data_set_version
         start_datetime
+        end_datetime
         error_msg
       }
       message
@@ -124,6 +125,13 @@ query dataProcessDetails($input: DataProcessDetailsInput){
             }
           }
         }
+        file_details {
+          file_name
+          status
+          start_time
+          end_time
+          file_size
+        }
       }
       message
     }

diff --git a/apiserver/graph/schema/dataprocessing.graphqls b/apiserver/graph/schema/dataprocessing.graphqls
@@ -57,6 +57,7 @@ input AddDataProcessInput {
 # 文件条目
 input FileItem {
   name: String!
+  size: String
 }
 
 # 数据处理配置条目
@@ -144,6 +145,8 @@ type DataProcessItem {
   post_data_set_version: String
   # 开始时间
   start_datetime: String!
+  # 结束时间
+  end_datetime: String!
   # 错误日志
   error_msg: String
 }
@@ -202,6 +205,7 @@ type DataProcessDetailsItem {
   error_msg: String
   data_process_config_info: [DataProcessConfigInfo!]
   config: [DataProcessConfig!]
+  file_details: [FileDetails!]
 }
 
 type DataProcessConfigInfo {
@@ -227,6 +231,15 @@ type DataProcessConfig {
   children: [DataProcessConfigChildren]
 }
 
+# 文件处理详情
+type FileDetails {
+  file_name: String!
+  status: String!
+  start_time: String!
+  end_time: String!
+  file_size: String!
+}
+
 # 数据处理配置项子项
 type DataProcessConfigChildren {
   name: String

diff --git a/deploy/charts/arcadia/templates/pg-init-data-configmap.yaml b/deploy/charts/arcadia/templates/pg-init-data-configmap.yaml
@@ -275,6 +275,7 @@ data:
             end_datetime character varying(64) COLLATE pg_catalog."default",
             status character varying(64) COLLATE pg_catalog."default",
             error_msg text COLLATE pg_catalog."default",
+            exc_msg text COLLATE pg_catalog."default",
             create_datetime character varying(64) COLLATE pg_catalog."default",
             create_user character varying(64) COLLATE pg_catalog."default",
             create_program character varying(64) COLLATE pg_catalog."default",
@@ -291,6 +292,7 @@ data:
         COMMENT ON COLUMN public.data_process_task_log.end_datetime IS '结束时间';
         COMMENT ON COLUMN public.data_process_task_log.status IS '状态';
         COMMENT ON COLUMN public.data_process_task_log.error_msg IS '错误信息';
+        COMMENT ON COLUMN public.data_process_task_log.exc_msg IS '异常信息';
         COMMENT ON COLUMN public.data_process_task_log.create_datetime IS '创建时间';
         COMMENT ON COLUMN public.data_process_task_log.create_user IS '创建人';
         COMMENT ON COLUMN public.data_process_task_log.create_program IS '创建程序';

diff --git a/pypi/data-processing/db-scripts/change/20240319-schema.sql b/pypi/data-processing/db-scripts/change/20240319-schema.sql
@@ -0,0 +1,3 @@
+ALTER TABLE data_process_task_log ADD COLUMN exc_msg text;
+
+COMMENT ON COLUMN data_process_task_log.exc_msg IS '异常信息';
diff --git a/pypi/data-processing/src/database_operate/data_process_db_operate.py b/pypi/data-processing/src/database_operate/data_process_db_operate.py
@@ -37,6 +37,7 @@ def list_by_page(req_json, pool):
           dpt.post_data_set_name,
           dpt.post_data_set_version,
           dpt.start_datetime,
+          dpt.end_datetime,
           dptl.error_msg
         from
           public.data_process_task dpt

diff --git a/pypi/data-processing/src/service/data_process_service.py b/pypi/data-processing/src/service/data_process_service.py
@@ -154,6 +154,10 @@ def info_by_id(req_json, pool):
 
     data["config"] = config_list_for_result
 
+    data["file_details"] = _set_file_status(
+        data.get("file_names"), task_id=id, conn_pool=pool
+    )
+
     logger.debug(f"{log_tag_const.DATA_PROCESS_DETAIL} The response data is: \n{data}")
 
     return {"status": 200, "message": "", "data": data}
@@ -265,6 +269,7 @@ def _get_and_set_basic_detail_info(from_result, task_id, conn_pool):
         from_result["name"] = detail_info_data["name"]
         from_result["status"] = detail_info_data["status"]
         from_result["file_type"] = detail_info_data["file_type"]
+        from_result["file_names"] = detail_info_data["file_names"]
         from_result["file_num"] = file_num
         from_result["pre_dataset_name"] = detail_info_data["pre_data_set_name"]
         from_result["pre_dataset_version"] = detail_info_data["pre_data_set_version"]
@@ -763,3 +768,18 @@ def _get_document_chunk_preview(task_id, conn_pool):
             )
 
     return chunk_list_preview
+
+
+def _set_file_status(file_names, task_id, conn_pool):
+    detail_info_params = {"task_id": task_id}
+    res = data_process_document_db_operate.list_file_by_task_id(
+        detail_info_params, pool=conn_pool
+    )
+
+    documents = res.get("data")
+    for document in documents:
+        for item in file_names:
+            if item.get("name") == document.get("file_name"):
+                document["file_size"] = item.get("size")
+
+    return documents