feat(Build): use bench get-app cache

press/api/tests/test_bench.py

@@ -80,17 +80,6 @@ def test_new_fn_creates_release_group_awaiting_deploy_when_called_by_press_admin
 		self.assertEqual(get_res["status"], "Awaiting Deploy")
 		self.assertEqual(get_res["public"], False)
 
-	def _set_press_settings_for_docker_build(self):
-		press_settings = create_test_press_settings()
-		cwd = os.getcwd()
-		back = os.path.join(cwd, "..")
-		bench_dir = os.path.abspath(back)
-		build_dir = os.path.join(bench_dir, "test_builds")
-		clone_dir = os.path.join(bench_dir, "test_clones")
-		press_settings.db_set("build_directory", build_dir)
-		press_settings.db_set("clone_directory", clone_dir)
-		press_settings.db_set("docker_registry_url", "registry.local.frappe.dev")
-
 	@patch(
 		"press.press.doctype.deploy_candidate.deploy_candidate.frappe.enqueue_doc",
 		new=foreground_enqueue_doc,
@@ -105,7 +94,7 @@ def test_deploy_fn_deploys_bench_container(self):
 		release.status = "Approved"
 		release.save()
 
-		self._set_press_settings_for_docker_build()
+		set_press_settings_for_docker_build()
 		frappe.set_user(self.team.user)
 		group = new(
 			{
@@ -120,10 +109,7 @@ def test_deploy_fn_deploys_bench_container(self):
 
 		dc_count_before = frappe.db.count("Deploy Candidate", filters={"group": group})
 		d_count_before = frappe.db.count("Deploy", filters={"group": group})
-		DeployCandidate.command = "docker buildx build"
-		DeployCandidate.command += (
-			" --cache-from type=gha --cache-to type=gha,mode=max --load"
-		)
+		patch_dc_command_for_ci()
 		deploy(group, [{"app": self.app.name}])
 		dc_count_after = frappe.db.count("Deploy Candidate", filters={"group": group})
 		d_count_after = frappe.db.count("Deploy", filters={"group": group})
@@ -641,3 +627,20 @@ def test_list_tagged_benches(self):
 		self.assertEqual(
 			all(bench_filter={"status": "", "tag": "test_tag"}), [self.bench_with_tag_dict]
 		)
+
+
+def set_press_settings_for_docker_build() -> None:
+	press_settings = create_test_press_settings()
+	cwd = os.getcwd()
+	back = os.path.join(cwd, "..")
+	bench_dir = os.path.abspath(back)
+	build_dir = os.path.join(bench_dir, "test_builds")
+	clone_dir = os.path.join(bench_dir, "test_clones")
+	press_settings.db_set("build_directory", build_dir)
+	press_settings.db_set("clone_directory", clone_dir)
+	press_settings.db_set("docker_registry_url", "registry.local.frappe.dev")
+
+
+def patch_dc_command_for_ci():
+	DeployCandidate.command = "docker buildx build"
+	DeployCandidate.command += " --cache-from type=gha --cache-to type=gha,mode=max --load"

press/docker/Dockerfile

@@ -219,7 +219,14 @@ COPY --chown=frappe:frappe common_site_config.json /home/frappe/frappe-bench/sit
 {% for app in doc.apps %}
 {% if app.app != "frappe" %}
 
-RUN --mount=type=cache,target=/home/frappe/.cache,uid=1000,gid=1000 --mount=type=bind,source=apps/{{ app.app }},target=/home/frappe/context/apps/{{ app.app }} bench get-app file:///home/frappe/context/apps/{{ app.app }} `#stage-apps-{{ app.app }}`
+RUN --mount=type=cache,target=/home/frappe/.cache,uid=1000,gid=1000 \
+  --mount=type=bind,source=apps/{{ app.app }},target=/home/frappe/context/apps/{{ app.app }} \
+  bench get-app file:///home/frappe/context/apps/{{ app.app }} \
+    {% if doc.use_app_cache %}
+    # Bench get-app flags to use get-app cache
+    --cache-key {{ app.hash }} {% if doc.compress_app_cache %}--compress-artifacts{% endif %} \
+    {% endif %}
+  `#stage-apps-{{ app.app }}`
 
 {% endif %}
 {% endfor %}

press/press/doctype/app/app.py

@@ -4,8 +4,12 @@
 
 
 import frappe
+import typing
 from frappe.model.document import Document
 
+if typing.TYPE_CHECKING:
+	from press.press.doctype.app_source.app_source import AppSource
+
 
 class App(Document):
 	def add_source(
@@ -16,7 +20,7 @@
 		team=None,
 		github_installation_id=None,
 		public=False,
-	):
+	) -> "AppSource":
 		existing_source = frappe.get_all(
 			"App Source",
 			{"app": self.name, "repository_url": repository_url, "branch": branch, "team": team},

press/press/doctype/app_release/test_app_release.py

@@ -3,12 +3,16 @@
 # See license.txt
 
 import unittest
+import typing
 import frappe
 
 from press.press.doctype.app_source.app_source import AppSource
 
+if typing.TYPE_CHECKING:
+	from press.press.doctype.app_release.app_release import AppRelease
 
-def create_test_app_release(app_source: AppSource, hash: str = None):
+
+def create_test_app_release(app_source: AppSource, hash: str = None) -> "AppRelease":
 	"""Create test app release given App source."""
 	hash = hash or frappe.mock("sha1")
 	app_release = frappe.get_doc(

press/press/doctype/bench_get_app_cache/bench_get_app_cache.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2024, Frappe and contributors
+// For license information, please see license.txt
+
+// frappe.ui.form.on("Bench Get App Cache", {
+// 	refresh(frm) {
+
+// 	},
+// });

press/press/doctype/bench_get_app_cache/bench_get_app_cache.json

@@ -0,0 +1,102 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2024-02-01 16:42:30.143539",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "tab_2_tab",
+  "file_name",
+  "size",
+  "accessed",
+  "column_break_uoai",
+  "app",
+  "is_compressed",
+  "section_break_fbua",
+  "raw"
+ ],
+ "fields": [
+  {
+   "fieldname": "tab_2_tab",
+   "fieldtype": "Tab Break",
+   "label": "Tab 2"
+  },
+  {
+   "fieldname": "file_name",
+   "fieldtype": "Data",
+   "label": "File Name",
+   "read_only": 1
+  },
+  {
+   "fieldname": "size",
+   "fieldtype": "Float",
+   "in_list_view": 1,
+   "label": "Size (MB)",
+   "read_only": 1
+  },
+  {
+   "fieldname": "accessed",
+   "fieldtype": "Datetime",
+   "in_list_view": 1,
+   "label": "Accessed",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_uoai",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "app",
+   "fieldtype": "Data",
+   "in_list_view": 1,
+   "label": "App",
+   "read_only": 1
+  },
+  {
+   "default": "0",
+   "fieldname": "is_compressed",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Is Compressed",
+   "read_only": 1
+  },
+  {
+   "fieldname": "section_break_fbua",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "raw",
+   "fieldtype": "Code",
+   "label": "Raw",
+   "read_only": 1
+  }
+ ],
+ "in_create": 1,
+ "index_web_pages_for_search": 1,
+ "is_virtual": 1,
+ "links": [],
+ "modified": "2024-02-02 18:54:26.533734",
+ "modified_by": "Administrator",
+ "module": "Press",
+ "name": "Bench Get App Cache",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "accessed",
+ "sort_order": "DESC",
+ "states": []
+}

press/press/doctype/bench_get_app_cache/bench_get_app_cache.py

@@ -0,0 +1,119 @@
+# Copyright (c) 2024, Frappe and contributors
+# For license information, please see license.txt
+
+from datetime import datetime
+
+import frappe
+from frappe.model.document import Document
+from press.press.doctype.deploy_candidate.cache_utils import run_command_in_docker_cache
+from press.utils import ttl_cache
+
+
+class BenchGetAppCache(Document):
+	@staticmethod
+	def get_data():
+		data = get_app_cache_items()
+		return data
+
+	def load_from_db(self):
+		db = {v.name: v for v in BenchGetAppCache.get_data()}
+		return super(Document, self).__init__(db[self.name])
+
+	def delete(self):
+		run_command_in_docker_cache(f"rm bench/apps/{self.file_name}")
+		get_app_cache_items.cache.invalidate()
+
+	@staticmethod
+	def clear_app_cache() -> None:
+		run_command_in_docker_cache("rm bench/apps/*.tar bench/apps/*.tgz")
+		get_app_cache_items.cache.invalidate()
+
+	@staticmethod
+	def clear_app_cache_by_app(app: str) -> None:
+		run_command_in_docker_cache(f"rm bench/apps/{app}-*.tar bench/apps/{app}-*.tgz")
+		get_app_cache_items.cache.invalidate()
+
+	@staticmethod
+	def get_list(_):
+		return BenchGetAppCache.get_data()
+
+	@staticmethod
+	def get_count(_):
+		data = BenchGetAppCache.get_data()
+		return len(data)
+
+	"""
+	The methods below are not applicable hence no-op.
+	"""
+
+	def db_update(self):
+		pass
+
+	def db_insert(self, *args, **kwargs):
+		pass
+
+	@staticmethod
+	def get_stats(args):
+		return {}
+
+
+"""
+ttl_cache used cause checking app cache involves
+building an image to execute `ls` during build time
+this takes a few of seconds (mostly a minute).
+"""
+
+
+@ttl_cache(ttl=20)
+def get_app_cache_items():
+
+	result = run_command_in_docker_cache("ls -luAt --time-style=full-iso bench/apps")
+	if result["returncode"]:
+		return []
+
+	output = result["output"]
+	values = []
+
+	"""
+	# Example Output :
+	total 587164
+	-rw-r--r-- 1 1000 1000 251607040 2024-02-01 10:03:33.972950013 +0000 builder-13a6ece9dd.tar
+	-rw-r--r-- 1 1000 1000 321587200 2024-02-01 10:01:04.109586013 +0000 hrms-84aced29ec.tar
+	-rw-r--r-- 1 1000 1000  28057600 2024-02-01 10:00:11.669851002 +0000 wiki-8b369c63dd.tar
+	"""
+
+	for line in output.splitlines():
+		doc = get_dict_from_ls_line(line)
+		if doc is not None:
+			values.append(doc)
+	return values
+
+
+def get_dict_from_ls_line(line: str):
+	parts = [p for p in line.split(" ") if p]
+	if len(parts) != 9:
+		return None
+
+	size = 0
+	accessed = datetime.fromtimestamp(0)
+	datestring = " ".join(parts[5:8])
+	try:
+		size = int(parts[4]) / 1_000_000
+		accessed = datetime.fromisoformat(datestring)
+	except ValueError:
+		"""
+		Invalid values passed above ∵ format not as expected. Use field `raw`
+		to debug and fix. Erroring out will prevent clearing of cache.
+		"""
+		pass
+
+	file_name = parts[-1]
+	return frappe._dict(
+		name=file_name.split(".", 1)[0],
+		file_name=file_name,
+		app=file_name.split("-")[0],
+		is_compressed=file_name.endswith(".tgz"),
+		size=size,
+		accessed=accessed,
+		raw=line,
+	)

press/press/doctype/bench_get_app_cache/test_bench_get_app_cache.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2024, Frappe and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestBenchGetAppCache(FrappeTestCase):
+	pass