Exclude courses which have the download button disabled from mirror drives

content_sync/constants.py

@@ -19,3 +19,8 @@
 DEV_DRAFT_URL = "http://10.1.0.102:8044"
 DEV_LIVE_URL = "http://10.1.0.102:8045"
 DEV_TEST_URL = "http://10.1.0.102:8046"
+
+
+# Publish Date Constants
+PUBLISH_DATE_LIVE = "publish_date"
+PUBLISH_DATE_DRAFT = "draft_publish_date"

content_sync/pipelines/definitions/concourse/mass_build_sites.py

@@ -90,7 +90,7 @@ def __init__(  # noqa: PLR0913
         hugo_arg_overrides: Optional[str] = None,
     ):
         vars = get_common_pipeline_vars()  # noqa: A001
-        sites = list(get_publishable_sites(version))
+        sites = list(get_publishable_sites(version, is_offline=offline))
         shuffle(sites)
         self.sites = sites
         self.version = version

content_sync/tasks.py

@@ -20,6 +20,7 @@
 from content_sync.constants import VERSION_DRAFT, VERSION_LIVE, WEBSITE_LISTING_DIRPATH
 from content_sync.decorators import single_task
 from content_sync.models import ContentSyncState
+from content_sync.utils import get_publishable_sites
 from main.celery import app
 from main.s3_utils import get_boto3_resource
 from websites.api import (
@@ -460,13 +461,14 @@ def update_websites_in_root_website():
     if settings.CONTENT_SYNC_BACKEND:
         root_website = Website.objects.get(name=settings.ROOT_WEBSITE_NAME)
         # Get all sites, minus any sites that have never been successfully published
-        sites = Website.objects.exclude(
-            Q(**{"draft_publish_date__isnull": True})
-            & Q(**{"publish_date__isnull": True})
-        )
-        sites = sites.exclude(Q(url_path__isnull=True))
+        # and have downloading disabled
+        sites = get_publishable_sites(is_offline=True)
         # Exclude the root website
         sites = sites.exclude(name=settings.ROOT_WEBSITE_NAME)
+
+        # Remove the content for unpublished or not downloadable sites
+        WebsiteContent.objects.exclude(website__in=sites).delete()
+
         fields = [
             "website",
             "type",

content_sync/utils.py

@@ -22,14 +22,15 @@
     OFFLINE_START,
     ONLINE_END,
     ONLINE_START,
+    PUBLISH_DATE_DRAFT,
+    PUBLISH_DATE_LIVE,
     START_TAG_PREFIX,
     TARGET_OFFLINE,
     VERSION_DRAFT,
-    VERSION_LIVE,
 )
 from main.s3_utils import get_boto3_resource
 from main.utils import is_dev
-from websites.constants import WEBSITE_CONTENT_FILETYPE
+from websites.constants import CONTENT_TYPE_METADATA, WEBSITE_CONTENT_FILETYPE
 from websites.models import Website, WebsiteContent
 from websites.site_config_api import SiteConfig
 
@@ -158,24 +159,42 @@ def get_ocw_studio_api_url():
     return "http://10.1.0.102:8043" if is_dev() else settings.SITE_BASE_URL
 
 
-def get_publishable_sites(version: str):
+def get_publishable_sites(version: str = "", *, is_offline: bool = False):
     """
     Get a QuerySet of Website objects that are eligible for publishing
 
     Args:
         version(str): The version (draft/live) to check publish eligibility with
+        is_offline(bool): Is the build type offline or not
     """
-    publish_date_field = (
-        "publish_date" if version == VERSION_LIVE else "draft_publish_date"
-    )
+    publish_date_field_filter = {
+        f"{PUBLISH_DATE_DRAFT}__isnull": True,
+        f"{PUBLISH_DATE_LIVE}__isnull": True,
+    }
+
+    if version:
+        publish_date_field_filter.pop(
+            f"{PUBLISH_DATE_LIVE}__isnull"
+            if version == VERSION_DRAFT
+            else f"{PUBLISH_DATE_DRAFT}__isnull"
+        )
+
     # Get all sites, minus any sites that have never been successfully published and test sites  # noqa: E501
     sites = (
         Website.objects.exclude(
-            Q(**{f"{publish_date_field}__isnull": True}) | Q(url_path__isnull=True)
+            Q(**publish_date_field_filter) | Q(url_path__isnull=True)
         )
         .exclude(unpublish_status__isnull=False)
         .exclude(name__in=settings.OCW_TEST_SITE_SLUGS)
     )
+
+    if is_offline:
+        # filter sites with download disabled
+        download_disabled_sites = WebsiteContent.objects.filter(
+            type=CONTENT_TYPE_METADATA, metadata__hide_download=True
+        ).values_list("website__short_id", flat=True)
+        sites = sites.exclude(short_id__in=download_disabled_sites)
+
     return sites.prefetch_related("starter")
 
 

content_sync/utils_test.py

@@ -40,6 +40,7 @@
 )
 from main.s3_utils import get_boto3_client
 from ocw_import.conftest import MOCK_BUCKET_NAME, setup_s3
+from websites.constants import CONTENT_TYPE_METADATA
 from websites.factories import WebsiteContentFactory, WebsiteStarterFactory
 from websites.site_config_api import ConfigItem, SiteConfig
 
@@ -257,7 +258,8 @@ def test_get_ocw_studio_api_url(settings, mocker, is_dev):
 
 
 @pytest.mark.parametrize("version", [VERSION_DRAFT, VERSION_LIVE])
-def test_get_publishable_sites(settings, mocker, mass_build_websites, version):
+@pytest.mark.parametrize("offline", [False, True])
+def test_get_publishable_sites(settings, mocker, mass_build_websites, version, offline):
     """get_publishable_sites should return a queryset of sites that have been published before"""
     unpublished_site = mass_build_websites[0]
     if version == VERSION_DRAFT:
@@ -270,8 +272,19 @@ def test_get_publishable_sites(settings, mocker, mass_build_websites, version):
         test_site.name = test_site_slug
         test_site.save()
     assert len(mass_build_websites) == 7
-    publishable_sites = get_publishable_sites(version)
-    assert publishable_sites.count() == 4
+
+    publishable_sites_count = 4
+
+    if offline:
+        WebsiteContentFactory.create(
+            type=CONTENT_TYPE_METADATA,
+            metadata={"hide_download": True},
+            website=mass_build_websites[4],
+        )
+        publishable_sites_count = 3
+
+    publishable_sites = get_publishable_sites(version, is_offline=offline)
+    assert publishable_sites.count() == publishable_sites_count
 
 
 @pytest.mark.parametrize("version", [VERSION_DRAFT, VERSION_LIVE])