prod (#511)

* cleanup and fix docker local builds (#473)

* Add routes for care scribe (#493)

* fix: Adds Linter CI and maintains formatting across codebase (#497)

* ci(lint): added linter CI

Signed-off-by: Abhiuday <contact.abhiuday@gmail.com>

* fix(pre-commit): updates the linter version

Signed-off-by: Abhiuday <contact.abhiuday@gmail.com>

* fix(ayushma): fixed format across codebase

Signed-off-by: Abhiuday <contact.abhiuday@gmail.com>

* ci(lint): fixed CI

Signed-off-by: Abhiuday <contact.abhiuday@gmail.com>

---------

Signed-off-by: Abhiuday <contact.abhiuday@gmail.com>

* fix(error): added more specific errors (#491)

Signed-off-by: Abhiuday <contact.abhiuday@gmail.com>

* Stability Enhancements and Bug Fixes (#510)

* Stability Enhancements and Bug Fixes

* Fix ai name removal

---------

Signed-off-by: Abhiuday <contact.abhiuday@gmail.com>
Co-authored-by: Aakash Singh <mail@singhaakash.dev>
Co-authored-by: Ashesh <3626859+Ashesh3@users.noreply.github.com>
Co-authored-by: Abhiuday Gupta <77210185+aeswibon@users.noreply.github.com>

.devcontainer/Dockerfile

@@ -10,6 +10,3 @@ ENV PYTHONUNBUFFERED 1
 # [Optional] Uncomment this section to install additional OS packages.
 # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 #     && apt-get -y install --no-install-recommends <your-package-list-here>
-
-
-

.github/workflows/linter.yaml

@@ -0,0 +1,32 @@
+name: Lint Code Base
+on:
+  pull_request:
+    branches:
+      - master
+jobs:
+  build:
+    if: github.repository == 'coronasafe/ayushma'
+    name: Lint Code Base
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+      statuses: write
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Lint Code Base
+        uses: super-linter/super-linter/slim@v6
+        env:
+          DEFAULT_BRANCH: master
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VALIDATE_ALL_CODEBASE: false
+          VALIDATE_PYTHON_BLACK: true
+          VALIDATE_PYTHON_FLAKE8: true
+          VALIDATE_PYTHON_ISORT: true
+          LINTER_RULES_PATH: /
+          PYTHON_FLAKE8_CONFIG_FILE: "setup.cfg"
+          PYTHON_BLACK_CONFIG_FILE: "pyproject.toml"
+          PYTHON_ISORT_CONFIG_FILE: "pyproject.toml"

.pre-commit-config.yaml

@@ -1,26 +1,26 @@
 exclude: "docs|node_modules|migrations|.git|.venv"
 default_stages: [commit]
 fail_fast: true
-
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.5.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
-
+        additional_dependencies: ["isort[pyproject]"]
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 24.3.0
     hooks:
       - id: black
-
+        args: ["--config=pyproject.toml"]
   - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
+    rev: 7.0.0
     hooks:
       - id: flake8
         args: ["--config=setup.cfg"]
+        additional_dependencies: [flake8-isort]

.vscode/settings.json

@@ -10,7 +10,7 @@
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter",
     "editor.codeActionsOnSave": {
-      "source.organizeImports": true
+      "source.organizeImports": "explicit"
     }
   },
   "editor.formatOnSave": true,

Makefile

@@ -31,7 +31,7 @@ logs:
 
 makemigrations: up
 	docker exec django bash -c "python manage.py makemigrations"
-	
+
 checkmigration:
 	docker compose -f $(docker_config_file) exec django bash -c "python manage.py makemigrations --check --dry-run"
 

Pipfile

@@ -53,6 +53,7 @@ factory-boy = "==3.3.0"
 django-debug-toolbar = "==4.2.0"
 django-extensions = "==3.2.3"
 psycopg = {extras = ["binary"], version = "==3.1.17"}
+watchdog = "==4.0.0"
 
 [requires]
 python_version = "3.11"

ayushma/admin.py

@@ -1,7 +1,6 @@
 from django.contrib import admin
 from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
 from djangoql.admin import DjangoQLSearchMixin
-from simple_history.admin import SimpleHistoryAdmin
 
 from ayushma.models.services import Service, TempToken
 from ayushma.models.testsuite import Feedback, TestQuestion, TestRun, TestSuite
@@ -32,7 +31,15 @@ class UserAdmin(DjangoQLSearchMixin, BaseUserAdmin):
     fieldsets = (
         (
             None,
-            {"fields": ("email", "username", "password", "external_id", "allow_key")},
+            {
+                "fields": (
+                    "email",
+                    "username",
+                    "password",
+                    "external_id",
+                    "allow_key",
+                )
+            },
         ),
         (
             "User info",

ayushma/apps.py

@@ -6,5 +6,5 @@ class AyushmaConfig(AppConfig):
     name = "ayushma"
 
     def ready(self):
-        #from .signals import   # noqa: F401
+        # from .signals import   # noqa: F401
         pass

ayushma/management/commands/upsert.py

@@ -31,7 +31,8 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         upsert_dir = "upsert"
         pinecone.init(
-            api_key=settings.PINECONE_API_KEY, environment=settings.PINECONE_ENVIRONMENT
+            api_key=settings.PINECONE_API_KEY,
+            environment=settings.PINECONE_ENVIRONMENT,
         )
         print("Initialized Pinecone and OpenAI")
 
@@ -49,7 +50,7 @@ def handle(self, *args, **options):
 
             batch_size = 100  # process everything in batches of 100 (creates 100 vectors per upset)
 
-            print(f"Fetching Pinecone index...")
+            print("Fetching Pinecone index...")
             if settings.PINECONE_INDEX not in pinecone.list_indexes():
                 pinecone.create_index(
                     settings.PINECONE_INDEX,

ayushma/migrations/0052_document_failed.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.6 on 2024-04-17 13:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ayushma", "0051_project_tts_engine"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="document",
+            name="failed",
+            field=models.BooleanField(default=False),
+        ),
+    ]

ayushma/models/document.py

@@ -19,6 +19,7 @@ class Document(BaseModel):
         "TestQuestion", on_delete=models.PROTECT, null=True, blank=True
     )
     uploading = models.BooleanField(default=True)
+    failed = models.BooleanField(default=False)
 
     def __str__(self) -> str:
         return f"{self.title} in {self.project.title}"

ayushma/models/enums.py

@@ -18,6 +18,13 @@ class STTEngine(IntegerChoices):
     GOOGLE = 2
     SELF_HOSTED = 3
 
+    @classmethod
+    def get_id_from_name(cls, name):
+        for member in cls:
+            if member.name.lower() == name.lower():
+                return member.value
+        return None
+
 
 class TTSEngine(IntegerChoices):
     OPENAI = (1, "openai")

ayushma/models/services.py

@@ -1,5 +1,4 @@
 import secrets
-import time
 
 from django.db import models
 from django.utils import timezone

ayushma/models/token.py

@@ -1,6 +1,6 @@
+from django.conf import settings
 from django.db import models
 from django.utils.translation import gettext_lazy as _
-from django.conf import settings
 
 from ayushma.token import RandomStringTokenGenerator
 from utils.models.base import BaseModel
@@ -11,9 +11,7 @@ class ResetPasswordToken(BaseModel):
         settings.AUTH_USER_MODEL,
         related_name="password_reset_tokens",
         on_delete=models.CASCADE,
-        verbose_name=_(
-            "The User which is associated to this password reset token"
-        ),
+        verbose_name=_("The User which is associated to this password reset token"),
     )
     created_at = models.DateTimeField(
         auto_now_add=True, verbose_name=_("When was this token generated")

ayushma/permissions.py

@@ -1,5 +1,3 @@
-from ipaddress import ip_address
-
 from django.utils import timezone
 from rest_framework import permissions
 

ayushma/serializers/document.py

@@ -1,12 +1,6 @@
-import os
-import uuid
-
-from django.conf import settings
-from django.core.files.storage import FileSystemStorage
 from rest_framework import serializers
-from rest_framework.response import Response
 
-from ayushma.models import Document, DocumentType
+from ayushma.models import Document
 
 
 class DocumentSerializer(serializers.ModelSerializer):
@@ -23,7 +17,12 @@ class Meta:
             "text_content",
             "uploading",
         )
-        read_only_fields = ("external_id", "created_at", "modified_at", "uploading")
+        read_only_fields = (
+            "external_id",
+            "created_at",
+            "modified_at",
+            "uploading",
+        )
         write_only_fields = ("file",)
 
     def validate(self, data):

ayushma/serializers/testsuite.py

@@ -9,7 +9,7 @@
     TestSuite,
 )
 from ayushma.serializers.document import DocumentSerializer
-from ayushma.serializers.project import ProjectSerializer, ProjectUpdateSerializer
+from ayushma.serializers.project import ProjectUpdateSerializer
 from ayushma.serializers.users import UserSerializer
 
 
@@ -67,7 +67,12 @@ class Meta:
             "modified_at",
         )
 
-        read_only_fields = ("user_object", "external_id", "created_at", "modified_at")
+        read_only_fields = (
+            "user_object",
+            "external_id",
+            "created_at",
+            "modified_at",
+        )
 
 
 class TestResultSerializer(serializers.ModelSerializer):

ayushma/serializers/users.py

@@ -16,6 +16,7 @@ class Meta:
             "allow_key",
             "is_staff",
             "is_reviewer",
+            "date_joined",
         )
 
 
@@ -56,6 +57,7 @@ class Meta:
             "is_staff",
             "is_reviewer",
             "password",
+            "date_joined",
         )
         read_only_fields = (
             "external_id",
@@ -64,6 +66,7 @@ class Meta:
             "allow_key",
             "is_staff",
             "is_reviewer",
+            "date_joined",
         )
 
     def update(self, instance, validated_data):

ayushma/tasks/__init__.py

@@ -0,0 +1,18 @@
+from celery import current_app
+from celery.schedules import crontab
+
+from ayushma.tasks.stale_cleanup import clean_stale_test_runs, clean_stale_upsert_doc
+
+
+@current_app.on_after_finalize.connect
+def setup_periodic_tasks(sender, **kwargs):
+    sender.add_periodic_task(
+        crontab(minute="*/30"),  # Every 30 minutes
+        clean_stale_test_runs.s(),
+        name="clean_stale_test_runs",
+    )
+    sender.add_periodic_task(
+        crontab(minute="*/30"),  # Every 30 minutes
+        clean_stale_upsert_doc.s(),
+        name="clean_stale_upsert_doc",
+    )

ayushma/tasks/stale_cleanup.py

@@ -0,0 +1,52 @@
+from datetime import timedelta
+
+from celery import shared_task
+from django.utils.timezone import now
+
+from ayushma.models.document import Document
+from ayushma.models.enums import StatusChoices
+from ayushma.models.testsuite import TestRun
+
+
+@shared_task(bind=True)
+def clean_stale_test_runs(self):
+    try:
+        # Get testRuns that are created over 6 hours ago and are still in RUNNING state
+        test_runs = TestRun.objects.filter(
+            created_at__lt=now() - timedelta(hours=6),
+            status=StatusChoices.RUNNING,
+        )
+
+        # Cancel the testRuns
+        for test_run in test_runs:
+            print(
+                f"Cleaning stale test run: {test_run.id}; Created at: {test_run.created_at}"
+            )
+            test_run.status = StatusChoices.FAILED
+            test_run.save()
+    except Exception as e:
+        print(f"Error occurred while cleaning stale test runs: {e}")
+        raise e
+
+
+@shared_task(bind=True)
+def clean_stale_upsert_doc(self):
+    try:
+        # Get stale Document objects that are still in UPLOADING state after 6 hours
+        documents = Document.objects.filter(
+            created_at__lt=now() - timedelta(hours=6),
+            uploading=True,
+        )
+
+        # Set the documents to failed state
+        for document in documents:
+            print(
+                f"Cleaning stale document: {document.id}; Created at: {document.created_at}"
+            )
+            document.failed = True
+            document.uploading = False
+            document.save()
+
+    except Exception as e:
+        print(f"Error occurred while cleaning stale test runs: {e}")
+        raise e

ayushma/token.py

@@ -7,6 +7,4 @@ def __init__(self, length=6):
         self.length = length
 
     def generate(self):
-        return "".join(
-            [random.choice(string.digits) for _ in range(self.length)]
-        )
+        return "".join([random.choice(string.digits) for _ in range(self.length)])