init: automate adding joins

frappe · Jul 18, 2023 · f3447b7 · f3447b7
1 parent 40228b4
commit f3447b7
Show file tree

Hide file tree

Showing 5 changed files with 200 additions and 55 deletions.
diff --git a/frontend/src/datasource/useDataSource.js b/frontend/src/datasource/useDataSource.js
@@ -103,6 +103,7 @@ function getDataSourceResource(name) {
 		whitelistedMethods: {
 			enqueue_sync_tables: 'enqueue_sync_tables',
 			get_tables: 'get_tables',
+			get_columns: 'get_columns',
 		},
 	})
 }
@@ -116,3 +117,17 @@ async function getTableName(data_source, table) {
 	cached_tablenames[data_source + table] = name
 	return name
 }
+
+export async function getAllColumns(data_source) {
+	return await call('insights.api.get_all_columns', {
+		data_source: data_source,
+	})
+}
+
+export async function getJoinPath(data_source, tableA, tableB) {
+	return await call('insights.api.get_join_path', {
+		data_source: data_source,
+		table1: tableA,
+		table2: tableB,
+	})
+}
diff --git a/frontend/src/notebook/blocks/query/builder/ColumnSelector.vue b/frontend/src/notebook/blocks/query/builder/ColumnSelector.vue
@@ -1,7 +1,7 @@
 <script setup>
 import UsePopover from '@/components/UsePopover.vue'
-import { useDataSourceTable } from '@/datasource/useDataSource'
-import { computed, ref, watch } from 'vue'
+import { getAllColumns, getJoinPath } from '@/datasource/useDataSource'
+import { computed, ref } from 'vue'
 const props = defineProps({
 	data_source: String,
 	tables: Array,
@@ -13,34 +13,9 @@ const props = defineProps({
 })
 const emit = defineEmits(['update:modelValue'])
 
-const tables = ref([])
-watch(
-	() => props.tables,
-	async () => {
-		if (!props.tables?.length) return []
-		const tablePromises = []
-		props.tables.forEach((table) => {
-			if (!table.table) return Promise.resolve()
-			tablePromises.push(
-				useDataSourceTable({
-					data_source: props.data_source,
-					table: table.table,
-				})
-			)
-		})
-		tables.value = await Promise.all(tablePromises)
-	},
-	{ immediate: true }
-)
-
+const all_columns = await getAllColumns(props.data_source)
 const columns = computed(() => {
-	const localColumns = props.localColumns.filter(filterFn)
-	if (!tables.value?.length) return localColumns
-	const tableColumns = tables.value
-		.map((d) => d.doc?.columns)
-		.flat()
-		.filter(filterFn)
-	return localColumns.concat(tableColumns)
+	return props.localColumns.concat(all_columns).filter(filterFn)
 })
 function filterFn(col, currIndex, self) {
 	if (!col) return false
@@ -57,9 +32,12 @@ const columnOptions = computed(() => {
 	if (!columns.value?.length) return []
 	return columns.value.map((c) => {
 		return {
-			label: c.label || c.alias,
-			description: c.description || c.table,
+			label: c.label || c.alias || c.column,
+			table: c.table,
 			table_label: c.table_label,
+			column: c.column,
+			type: c.type,
+			description: c.description || c.table,
 			value: getColumnValue(c),
 		}
 	})
@@ -114,20 +92,22 @@ const filteredColumnOptionsGroupedByTable = computed(() => {
 		const options = columnOptionsGroupedByTable.value[table]
 		const filteredOptions = options.filter((o) => {
 			return (
-				o.label.toLowerCase().includes(columnSearchTerm.value.toLowerCase()) ||
-				o.value.toLowerCase().includes(columnSearchTerm.value.toLowerCase()) ||
-				o.description.toLowerCase().includes(columnSearchTerm.value.toLowerCase())
+				o.label?.toLowerCase().includes(columnSearchTerm.value.toLowerCase()) ||
+				o.value?.toLowerCase().includes(columnSearchTerm.value.toLowerCase()) ||
+				o.description?.toLowerCase().includes(columnSearchTerm.value.toLowerCase())
 			)
 		})
-		if (filteredOptions.length) filtered[table] = filteredOptions
+		if (filteredOptions.length) filtered[table] = filteredOptions.slice(0, 20)
 	})
 	return filtered
 })
 
 const trigger = ref(null)
 const columnSearchTerm = ref('')
 const columnPopover = ref(null)
-function handleColumnSelect(col) {
+async function handleColumnSelect(col) {
+	debugger
+	console.log(await getJoinPath(props.data_source, 'tabToDo', col.table))
 	column.value = col
 	columnSearchTerm.value = ''
 	columnPopover.value?.close()
@@ -144,14 +124,10 @@ function handleColumnSelect(col) {
 			<span> {{ column?.label || 'Pick a column' }} </span>
 		</div>
 		<UsePopover ref="columnPopover" v-if="trigger" :targetElement="trigger">
-			<div class="w-[12rem] rounded bg-white text-base shadow transition-[width]">
-				<div class="flex items-center rounded-t-md border-b bg-white">
-					<Input
-						iconLeft="search"
-						class="rounded-b-none border-none bg-transparent text-sm focus:shadow-none focus:outline-none focus:ring-0"
-						v-model="columnSearchTerm"
-						placeholder="Search column..."
-					/>
+			<div class="w-[12rem] rounded-lg border bg-white text-base shadow transition-[width]">
+				<div class="flex items-center rounded-t-md bg-white px-2">
+					<FeatherIcon name="search" class="h-4 w-4 text-gray-500" />
+					<input v-model="columnSearchTerm" placeholder="Search column..." />
 				</div>
 				<div class="max-h-48 overflow-y-auto text-sm">
 					<p
@@ -165,7 +141,7 @@ function handleColumnSelect(col) {
 					</p>
 					<div v-else v-for="table in Object.keys(filteredColumnOptionsGroupedByTable)">
 						<div
-							class="sticky top-0 flex items-center border-b bg-white px-2 py-1 text-gray-700"
+							class="sticky top-0 flex items-center border-y bg-white px-2 py-1 text-gray-700"
 						>
 							<FeatherIcon name="table" class="mr-1 h-3.5 w-3.5" />
 							<span class="flex-1 py-0.5 text-sm">
@@ -178,7 +154,7 @@ function handleColumnSelect(col) {
 								:class="column?.value === col.value ? 'bg-gray-100' : ''"
 								@click="handleColumnSelect(col)"
 							>
-								<span>{{ col.label }}</span>
+								<span>{{ col.label || col.value }}</span>
 							</div>
 						</div>
 					</div>

diff --git a/insights/api/__init__.py b/insights/api/__init__.py
@@ -577,3 +577,94 @@ def contact_team(message_type, message_content, is_critical=False):
     except Exception as e:
         frappe.log_error(e)
         frappe.throw("Something went wrong. Please try again later.")
+
+
+@frappe.whitelist()
+@redis_cache()
+def get_all_columns(data_source):
+    check_data_source_permission(data_source)
+    InsightsTable = frappe.qb.DocType("Insights Table")
+    InsightsTableColumn = frappe.qb.DocType("Insights Table Column")
+    return (
+        frappe.qb.from_(InsightsTable)
+        .left_join(InsightsTableColumn)
+        .on(InsightsTable.name == InsightsTableColumn.parent)
+        .select(
+            InsightsTable.table.as_("table"),
+            InsightsTable.label.as_("table_label"),
+            InsightsTable.data_source.as_("data_source"),
+            InsightsTable.is_query_based.as_("is_query_based"),
+            InsightsTableColumn.column.as_("column"),
+            InsightsTableColumn.label.as_("label"),
+            InsightsTableColumn.type.as_("type"),
+        )
+        .where((InsightsTable.data_source == data_source) & (InsightsTable.hidden == 0))
+        .orderby(InsightsTable.is_query_based)
+        .orderby(InsightsTable.label)
+        .run(as_dict=True)
+    )
+
+
+@frappe.whitelist()
+def get_join_path(data_source, table1, table2):
+    join_graph = make_join_graph(data_source)
+    if not join_graph.bfs(table1, table2):
+        return None
+
+    # list of tables that will be involved in the join
+    path = join_graph.shortest_path(table1, table2)
+
+    # find the columns that will be used for the join
+    # if path = ["Sales Invoice", "Sales Invoice Item", "Item"]
+    # ret = [{ "left_table": "Sales Invoice", "left_column": "item_code", "right_table": "Sales Invoice Item", "right_column": "item_code" }, ...]
+
+    joins = []
+    for i in range(len(path) - 1):
+        left_table = path[i]
+        right_table = path[i + 1]
+        left_table_name = frappe.db.get_value(
+            "Insights Table", {"data_source": data_source, "table": left_table}, "name"
+        )
+        left_table_doc = frappe.get_doc("Insights Table", left_table_name)
+        for link in left_table_doc.table_links:
+            if link.foreign_table == right_table:
+                joins.append(
+                    {
+                        "left_table": left_table,
+                        "left_column": link.primary_key,
+                        "right_table": right_table,
+                        "right_column": link.foreign_key,
+                    }
+                )
+                break
+    return joins
+
+
+# @redis_cache(ttl=60 * 60 * 24)
+def make_join_graph(data_source):
+    from insights.api.join_graph import Graph
+
+    g = Graph()
+    tables = frappe.get_all("Insights Table", {"data_source": data_source}, ["name", "table"])
+    tableByName = {table["name"]: table["table"] for table in tables}
+    table_links = frappe.get_all(
+        "Insights Table Link",
+        filters={"parent": ["in", list(tableByName.keys())]},
+        fields=["parent", "foreign_table", "primary_key", "foreign_key"],
+    )
+    for link in table_links:
+        primary_table = tableByName[link.parent]
+        # if primary_table == "tabDocType" or link.foreign_table == "tabDocType":
+        #     continue
+        g.add_node(primary_table)
+        g.add_node(link.foreign_table)
+        g.add_edge(
+            primary_table,
+            link.foreign_table,
+            metadata={
+                "primary_key": link.primary_key,
+                "foreign_key": link.foreign_key,
+            },
+        )
+
+    return g
diff --git a/insights/api/join_graph.py b/insights/api/join_graph.py
@@ -0,0 +1,69 @@
+from collections import deque
+
+
+class Graph:
+    def __init__(self):
+        self.nodes = []
+        self.edges = {}
+        self.metadata = {}
+
+    def add_node(self, node):
+        if node in self.nodes:
+            return
+        self.nodes.append(node)
+        self.edges[node] = []
+
+    def add_edge(self, node1, node2, metadata=None):
+        if node2 in self.edges[node1]:
+            return
+        self.edges[node1].append(node2)
+        if metadata:
+            self.metadata[(node1, node2)] = metadata
+
+    def __str__(self):
+        return str(self.edges)
+
+    def bfs(self, start, end):
+        queue = deque()
+        queue.append(start)
+        visited = set()
+        visited.add(start)
+        while queue:
+            node = queue.popleft()
+            if node == end:
+                return True
+            for neighbor in self.edges[node]:
+                if neighbor not in visited:
+                    queue.append(neighbor)
+                    visited.add(neighbor)
+        return False
+
+    def shortest_path(self, start, end):
+        queue = deque()
+        queue.append([start])
+        visited = set()
+        visited.add(start)
+        while queue:
+            path = queue.popleft()
+            node = path[-1]
+            if node == end:
+                return path
+            for neighbor in self.edges[node]:
+                if neighbor not in visited:
+                    queue.append(path + [neighbor])
+                    visited.add(neighbor)
+
+    def get_all_possible_path_from_source(self, source):
+        queue = deque()
+        queue.append([source])
+        visited = set()
+        visited.add(source)
+        while queue:
+            path = queue.popleft()
+            node = path[-1]
+            for neighbor in self.edges[node]:
+                if neighbor not in visited:
+                    queue.append(path + [neighbor])
+                    visited.add(neighbor)
+        visited.remove(source)
+        return visited
diff --git a/insights/insights/doctype/insights_data_source/insights_data_source.py b/insights/insights/doctype/insights_data_source/insights_data_source.py
@@ -25,9 +25,7 @@
 
 class InsightsDataSource(Document):
     def before_insert(self):
-        if self.is_site_db and frappe.db.exists(
-            "Insights Data Source", {"is_site_db": 1}
-        ):
+        if self.is_site_db and frappe.db.exists("Insights Data Source", {"is_site_db": 1}):
             frappe.throw("Only one site database can be configured")
 
     @frappe.whitelist()
@@ -57,9 +55,7 @@ def on_trash(self):
 
         linked_doctypes = ["Insights Table"]
         for doctype in linked_doctypes:
-            for name in frappe.db.get_all(
-                doctype, {"data_source": self.name}, pluck="name"
-            ):
+            for name in frappe.db.get_all(doctype, {"data_source": self.name}, pluck="name"):
                 frappe.delete_doc(doctype, name)
 
         track("delete_data_source")
@@ -123,9 +119,7 @@ def validate_remote_db_fields(self):
                 frappe.throw(f"{field} is mandatory for Database")
 
     def before_save(self):
-        self.status = (
-            SOURCE_STATUS.Active if self.test_connection() else SOURCE_STATUS.Inactive
-        )
+        self.status = SOURCE_STATUS.Active if self.test_connection() else SOURCE_STATUS.Inactive
 
     def test_connection(self, raise_exception=False):
         try: