chore: run pylint locally and fix lint issues

.pylintrc

@@ -52,7 +52,7 @@ ignore=CVS
 # ignore-list. The regex matches against paths and can be in Posix or Windows
 # format. Because '\\' represents the directory delimiter on Windows systems,
 # it can't be used as an escape character.
-ignore-paths=
+ignore-paths=./deploy,./examples
 
 # Files or directories matching the regular expression patterns are skipped.
 # The regex matches against base names, not paths. The default value ignores
@@ -337,7 +337,7 @@ indent-after-paren=4
 indent-string='    '
 
 # Maximum number of characters on a single line.
-max-line-length=100
+max-line-length=200
 
 # Maximum number of lines in a module.
 max-module-lines=1000
@@ -427,10 +427,15 @@ disable=raw-checker-failed,
         file-ignored,
         suppressed-message,
         useless-suppression,
-        deprecated-pragma,
-        use-symbolic-message-instead,
-        use-implicit-booleaness-not-comparison-to-string,
-        use-implicit-booleaness-not-comparison-to-zero
+        deprecated-pragma,redefined-outer-name,
+        use-symbolic-message-instead,missing-class-docstring,
+        missing-module-docstring,too-many-instance-attributes,
+        logging-fstring-interpolation,too-few-public-methods,
+        invalid-character-zero-width-space,missing-function-docstring,
+        duplicate-value,too-many-lines,dangerous-default-value,deprecated-method,
+        broad-exception-caught,redefined-builtin,c-extension-no-member,too-many-arguments,
+        too-many-branches,too-many-locals,too-many-statements,f-string-without-interpolation,
+        consider-using-enumerate,
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

Makefile

@@ -300,6 +300,17 @@ prepare-push: manifests generate fmt vet gql-gen
 	@go install github.com/swaggo/swag/cmd/swag@latest
 	@swag init -o apiserver/docs .
 
+PYTHON_INDEX_URL ?=https://pypi.mirrors.ustc.edu.cn/simple/
+.PHONY: prepare-push-pypi
+prepare-push-pypi:
+	@echo "install black"
+	@pip install pylint black isort -i ${PYTHON_INDEX_URL}
+	@echo "format python code"
+	@black .
+	@echo "sort python imports"
+	@isort .
+	@echo "run pylint on data-processing"
+	@pylint --rcfile .pylintrc ./**/*.py
 # Commands for Data-Processing
 DATA_PROCESSING_IMAGE ?= kubebb/dp-base
 

README.md

@@ -8,6 +8,12 @@
     <a href="https://goreportcard.com/report/github.com/kubeagi/arcadia">
       <img alt="Go Report Card" src="https://goreportcard.com/badge/kubeagi/arcadia?style=flat-square" />
     </a>
+    <a href="https://github.com/pylint-dev/pylint">
+      <img alt="Pylint Card" src="https://github.com/kubeagi/arcadia/actions/workflows/pypi_lint/badge.svg" />
+    </a>
+    <a href="https://github.com/psf/black">
+      <img alt="CodeStyle" src="https://img.shields.io/badge/code%20style-black-000000.svg" />
+    </a>
   </p>
 </div>
 

config/crd/kustomization.yaml

@@ -15,10 +15,10 @@ resources:
 - bases/arcadia.kubeagi.k8s.com.cn_applications.yaml
 - bases/chain.arcadia.kubeagi.k8s.com.cn_llmchains.yaml
 - bases/chain.arcadia.kubeagi.k8s.com.cn_retrievalqachains.yaml
+- bases/chain.arcadia.kubeagi.k8s.com.cn_apichains.yaml
 - bases/prompt.arcadia.kubeagi.k8s.com.cn_prompts.yaml
 - bases/retriever.arcadia.kubeagi.k8s.com.cn_knowledgebaseretrievers.yaml
 - bases/evaluation.arcadia.kubeagi.k8s.com.cn_rags.yaml
-- bases/chain.kubeagi.k8s.com.cn_apichains.yaml
 #+kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

deploy/charts/arcadia/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: arcadia
 description: A Helm chart(KubeBB Component) for KubeAGI Arcadia
 type: application
-version: 0.2.16
+version: 0.2.17
 appVersion: "0.1.0"
 
 keywords:

deploy/charts/arcadia/templates/dataprocess.yaml

@@ -37,7 +37,7 @@ spec:
           command:
             - "/bin/sh"
             - "-c"
-            - "python /arcadia_app/data_manipulation/server.py"
+            - "python /arcadia_app/src/server.py"
 ---
 apiVersion: v1
 kind: Service

deploy/llms/utils.py

@@ -1,6 +1,5 @@
 import asyncio
 import binascii
-from collections import defaultdict
 import contextlib
 import errno
 import functools
@@ -18,33 +17,22 @@
 import tempfile
 import threading
 import time
-from urllib.parse import urlencode, unquote, urlparse, parse_qsl, urlunparse
 import warnings
+from collections import defaultdict
 from inspect import signature
 from pathlib import Path
 from subprocess import list2cmdline
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Dict,
-    Optional,
-    Sequence,
-    Tuple,
-    Union,
-    Coroutine,
-    List,
-    Mapping,
-)
+from typing import (TYPE_CHECKING, Any, Coroutine, Dict, List, Mapping,
+                    Optional, Sequence, Tuple, Union)
+from urllib.parse import parse_qsl, unquote, urlencode, urlparse, urlunparse
 
 # Import psutil after ray so the packaged version is used.
 import psutil
-from google.protobuf import json_format
-
 import ray
 import ray._private.ray_constants as ray_constants
-from ray.core.generated.runtime_env_common_pb2 import (
-    RuntimeEnvInfo as ProtoRuntimeEnvInfo,
-)
+from google.protobuf import json_format
+from ray.core.generated.runtime_env_common_pb2 import \
+    RuntimeEnvInfo as ProtoRuntimeEnvInfo
 
 if TYPE_CHECKING:
     from ray.runtime_env import RuntimeEnv
@@ -281,9 +269,8 @@ def get_visible_accelerator_ids() -> Mapping[str, Optional[List[str]]]:
     to the visible ids."""
 
     from ray._private.accelerators import (
-        get_all_accelerator_resource_names,
         get_accelerator_manager_for_resource,
-    )
+        get_all_accelerator_resource_names)
 
     return {
         accelerator_resource_name: get_accelerator_manager_for_resource(
@@ -1535,10 +1522,10 @@ def check_version_info(cluster_metadata):
     # but for now, we don't have the same python version of fastchat and ray
     # so allow to use environment variable to overwrite them
     # make sure the overwritten version will be compatible
-    if os.getenv('RAY_VERSION') is not None:
-        version_info = (os.getenv('RAY_VERSION'), version_info[1])
-    if  os.getenv('PYTHON_VERSION') is not None:
-        version_info = (version_info[0], os.getenv('PYTHON_VERSION'))
+    if os.getenv("RAY_VERSION") is not None:
+        version_info = (os.getenv("RAY_VERSION"), version_info[1])
+    if os.getenv("PYTHON_VERSION") is not None:
+        version_info = (version_info[0], os.getenv("PYTHON_VERSION"))
     # ---------------------------- KubeAGI end --------------------------------
 
     if version_info != cluster_version_info:

examples/app-ui-using-streamlit/chat_app_english_teacher.py

@@ -1,15 +1,26 @@
-import streamlit as st
-import requests
 import os
 
+import requests
+import streamlit as st
+
 with st.sidebar:
-    server_url = st.text_input("服务 apiserver 请求地址, 默认为 http://arcadia-apiserver.kubeagi-system.svc:8081/chat", key="url")
-    conversion_id = st.text_input("如果想继续的话，可以输入上次的conversion_id，留空表示新对话", key="conversion_id")
+    server_url = st.text_input(
+        "服务 apiserver 请求地址, 默认为 http://arcadia-apiserver.kubeagi-system.svc:8081/chat",
+        key="url",
+    )
+    conversion_id = st.text_input(
+        "如果想继续的话，可以输入上次的conversion_id，留空表示新对话", key="conversion_id"
+    )
 
 st.title("💬 Chat with kubeagi")
 st.caption("🚀 A chatbot powered by Kubeagi")
 if "messages" not in st.session_state:
-    st.session_state["messages"] = [{"role": "assistant", "content": "Hello, I am English Teacher 🧑‍🏫 From KubeAGI 🤖"}]
+    st.session_state["messages"] = [
+        {
+            "role": "assistant",
+            "content": "Hello, I am English Teacher 🧑‍🏫 From KubeAGI 🤖",
+        }
+    ]
 
 if "first_show" not in st.session_state:
     st.session_state["first_show"] = True
@@ -21,15 +32,23 @@
     st.chat_message(msg["role"]).write(msg["content"])
 
 if prompt := st.chat_input():
-    response = requests.post(server_url,
-    json={"query":prompt,"response_mode":"blocking","conversion_id":conversion_id,"app_name":"base-chat-english-teacher", "app_namespace":"kubeagi-system"})
+    response = requests.post(
+        server_url,
+        json={
+            "query": prompt,
+            "response_mode": "blocking",
+            "conversion_id": conversion_id,
+            "app_name": "base-chat-english-teacher",
+            "app_namespace": "kubeagi-system",
+        },
+    )
     st.session_state.messages.append({"role": "user", "content": prompt})
     st.chat_message("user").write(prompt)
     msg = response.json()["message"]
     conversion_id = response.json()["conversion_id"]
 
     if st.session_state["first_show"]:
-        st.info('这次聊天的 conversion_id 是： '+conversion_id, icon="ℹ️")
+        st.info("这次聊天的 conversion_id 是： " + conversion_id, icon="ℹ️")
         st.session_state["first_show"] = False
 
     st.session_state.messages.append({"role": "assistant", "content": msg})

examples/app-ui-using-streamlit/chat_using_private_knowledgebase.py

@@ -1,15 +1,23 @@
-import streamlit as st
-import requests
 import os
 
+import requests
+import streamlit as st
+
 with st.sidebar:
-    server_url = st.text_input("服务 apiserver 请求地址, 默认为 http://arcadia-apiserver.kubeagi-system.svc:8081/chat", key="url")
-    conversion_id = st.text_input("如果想继续的话，可以输入上次的conversion_id，留空表示新对话", key="conversion_id")
+    server_url = st.text_input(
+        "服务 apiserver 请求地址, 默认为 http://arcadia-apiserver.kubeagi-system.svc:8081/chat",
+        key="url",
+    )
+    conversion_id = st.text_input(
+        "如果想继续的话，可以输入上次的conversion_id，留空表示新对话", key="conversion_id"
+    )
 
 st.title("💬 Chat with kubeagi")
 st.caption("🚀 A chatbot powered by Kubeagi")
 if "messages" not in st.session_state:
-    st.session_state["messages"] = [{"role": "assistant", "content": "您好，您可以问我任何关于考勤制度的问题，很高心为您服务。"}]
+    st.session_state["messages"] = [
+        {"role": "assistant", "content": "您好，您可以问我任何关于考勤制度的问题，很高心为您服务。"}
+    ]
 
 if "first_show" not in st.session_state:
     st.session_state["first_show"] = True
@@ -21,15 +29,23 @@
     st.chat_message(msg["role"]).write(msg["content"])
 
 if prompt := st.chat_input():
-    response = requests.post(server_url,
-    json={"query":prompt,"response_mode":"blocking","conversion_id":conversion_id,"app_name":"chat-with-kaoqin-kb", "app_namespace":"kubeagi-system"})
+    response = requests.post(
+        server_url,
+        json={
+            "query": prompt,
+            "response_mode": "blocking",
+            "conversion_id": conversion_id,
+            "app_name": "chat-with-kaoqin-kb",
+            "app_namespace": "kubeagi-system",
+        },
+    )
     st.session_state.messages.append({"role": "user", "content": prompt})
     st.chat_message("user").write(prompt)
     msg = response.json()["message"]
     conversion_id = response.json()["conversion_id"]
 
     if st.session_state["first_show"]:
-        st.info('这次聊天的 conversion_id 是： '+conversion_id, icon="ℹ️")
+        st.info("这次聊天的 conversion_id 是： " + conversion_id, icon="ℹ️")
         st.session_state["first_show"] = False
 
     st.session_state.messages.append({"role": "assistant", "content": msg})