From 14e2af531d50b78fe075261437d3e808528e5cff Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Fri, 28 Jul 2023 11:13:49 +0530 Subject: [PATCH 01/48] fix: pie chart tooltip --- frontend/src/widgets/Pie/Pie.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/widgets/Pie/Pie.vue b/frontend/src/widgets/Pie/Pie.vue index b40005077..c55529c0c 100644 --- a/frontend/src/widgets/Pie/Pie.vue +++ b/frontend/src/widgets/Pie/Pie.vue @@ -164,7 +164,7 @@ function appendPercentage(value) { }" /> Date: Fri, 28 Jul 2023 11:17:14 +0530 Subject: [PATCH 02/48] fix(test): version number '1.x.x' does not match PEP 440 rules --- insights/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/insights/__init__.py b/insights/__init__.py index 6c9eaec84..367626cc9 100644 --- a/insights/__init__.py +++ b/insights/__init__.py @@ -2,7 +2,7 @@ # For license information, please see license.txt -__version__ = "1.x.x" +__version__ = "1.0.0" def notify(**kwargs): From e0bb93c7b08d05e98ee42d8d2f909f2d597a3730 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Sat, 29 Jul 2023 00:12:46 +0530 Subject: [PATCH 03/48] refactor: extract `get_source_schema` method from query --- frontend/src/notebook/blocks/query/useQuery.js | 5 +++-- frontend/src/query/NativeQueryEditor.vue | 7 ++++++- frontend/src/query/useQueryResource.js | 2 -- frontend/src/utils/query/index.js | 1 - insights/api/__init__.py | 5 +++++ 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/frontend/src/notebook/blocks/query/useQuery.js b/frontend/src/notebook/blocks/query/useQuery.js index 7ccb3ce53..eb9342d15 100644 --- a/frontend/src/notebook/blocks/query/useQuery.js +++ b/frontend/src/notebook/blocks/query/useQuery.js @@ -120,8 +120,9 @@ function makeQuery(name) { watchDebounced(getUpdatedFields, setUnsaved, { deep: true, debounce: 500 }) state.fetchSourceSchema = async () => { - const response = await resource.get_source_schema.fetch() - state.sourceSchema = response.message + state.sourceSchema = await call('insights.api.get_source_schema', { + data_source: state.doc.data_source, + }) } state.loadChart = async () => { diff --git a/frontend/src/query/NativeQueryEditor.vue b/frontend/src/query/NativeQueryEditor.vue index 3cea965c5..fbb842f7a 100644 --- a/frontend/src/query/NativeQueryEditor.vue +++ b/frontend/src/query/NativeQueryEditor.vue @@ -1,9 +1,14 @@ diff --git a/frontend/src/query/ScriptQueryEditor.vue b/frontend/src/query/ScriptQueryEditor.vue new file mode 100644 index 000000000..591ece3c7 --- /dev/null +++ b/frontend/src/query/ScriptQueryEditor.vue @@ -0,0 +1,31 @@ + + + diff --git a/insights/insights/doctype/insights_query/insights_query.json b/insights/insights/doctype/insights_query/insights_query.json index 2874024af..5b11bbf0b 100644 --- a/insights/insights/doctype/insights_query/insights_query.json +++ b/insights/insights/doctype/insights_query/insights_query.json @@ -12,6 +12,7 @@ "is_native_query", "is_assisted_query", "is_saved_as_table", + "is_script_query", "column_break_3", "data_source", "is_stored", @@ -25,6 +26,7 @@ "limit", "section_break_cnma", "json", + "script", "query_and_result_tab", "section_break_11", "sql", @@ -109,7 +111,7 @@ "fieldtype": "Column Break" }, { - "depends_on": "eval: !doc.is_native_query", + "depends_on": "eval: !doc.is_assisted_query && !doc.is_native_query", "fieldname": "section_break_3", "fieldtype": "Section Break" }, @@ -120,7 +122,6 @@ "in_standard_filter": 1, "label": "Data Source", "options": "Insights Data Source", - "reqd": 1, "search_index": 1 }, { @@ -219,11 +220,24 @@ "fieldtype": "Check", "is_virtual": 1, "label": "Is Saved as Table" + }, + { + "default": "0", + "fieldname": "is_script_query", + "fieldtype": "Check", + "label": "Is Script Query" + }, + { + "depends_on": "eval: doc.is_script_query", + "fieldname": "script", + "fieldtype": "Code", + "label": "Script", + "options": "Python" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2023-07-22 23:42:07.915992", + "modified": "2023-07-28 21:59:23.884164", "modified_by": "Administrator", "module": "Insights", "name": "Insights Query", diff --git a/insights/insights/doctype/insights_query/insights_query.py b/insights/insights/doctype/insights_query/insights_query.py index d5c364f64..c3890f7df 100644 --- a/insights/insights/doctype/insights_query/insights_query.py +++ b/insights/insights/doctype/insights_query/insights_query.py @@ -24,6 +24,7 @@ ) from .insights_query_client import InsightsQueryClient from .insights_raw_query import InsightsRawQueryController +from .insights_script_query import InsightsScriptQueryController from .utils import ( CachedResults, InsightsChart, @@ -76,6 +77,8 @@ def variant_controller(self): return InsightsRawQueryController(self) if self.is_assisted_query: return InsightsAssistedQueryController(self) + if self.is_script_query: + return InsightsScriptQueryController(self) return InsightsLegacyQueryController(self) def validate(self): @@ -88,6 +91,7 @@ def reset(self): new_query.data_source = self.data_source new_query.is_native_query = self.is_native_query new_query.is_assisted_query = self.is_assisted_query + new_query.is_script_query = self.is_script_query new_query_dict = new_query.as_dict(no_default_fields=True) self.update(new_query_dict) self.status = Status.SUCCESS.value diff --git a/insights/insights/doctype/insights_query/insights_script_query.py b/insights/insights/doctype/insights_query/insights_script_query.py new file mode 100644 index 000000000..edfa493c3 --- /dev/null +++ b/insights/insights/doctype/insights_query/insights_script_query.py @@ -0,0 +1,79 @@ +# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + + +import frappe +from frappe.utils.safe_exec import safe_exec + +from .utils import get_columns_with_inferred_types + + +class InsightsScriptQueryController: + def __init__(self, doc): + self.doc = doc + + def validate(self): + pass + + def before_save(self): + pass + + def get_columns_from_results(self, results): + if not results: + return [] + return get_columns_with_inferred_types(results) + + def fetch_results(self): + script = self.doc.script + if not script: + return [] + + results = [] + try: + _locals = {"results": results} + safe_exec(script, None, _locals) + results = _locals["results"] + except BaseException: + frappe.log_error(title="Insights Script Query Error") + frappe.throw( + "There was an error executing the script. " + "Please check the error log for more details.", + title="Insights Script Query Error", + ) + + results = self.validate_results(results) + return results + + def validate_results(self, results): + if not results: + frappe.throw("The script should declare a variable named 'results'.") + + if not isinstance(results[0], list): + frappe.throw("Results should be a list of lists.") + + if not all(isinstance(row, list) for row in results): + frappe.throw("All rows should be lists.") + + if not all(isinstance(col, str) for col in results[0]): + frappe.throw("All columns should be strings.") + + if not all(len(row) == len(results[0]) for row in results): + frappe.throw("All rows should have the same number of columns.") + + if not all(col for col in results[0]): + frappe.throw("All columns should have a label.") + + results[0] = [{"label": col.strip()} for col in results[0]] + return results + + def before_fetch(self): + return + + def after_fetch(self, results): + return results + + def get_tables_columns(self): + return [] + + def get_selected_tables(self): + return [] From 8811109bd62f71fa11e09d9f7ea32b98f9a8e066 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Sat, 29 Jul 2023 20:43:48 +0530 Subject: [PATCH 06/48] chore: show more table options to filter --- .../notebook/blocks/query/builder/SourceAndTableSelector.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/notebook/blocks/query/builder/SourceAndTableSelector.vue b/frontend/src/notebook/blocks/query/builder/SourceAndTableSelector.vue index 3e69ce41f..ffe02e30d 100644 --- a/frontend/src/notebook/blocks/query/builder/SourceAndTableSelector.vue +++ b/frontend/src/notebook/blocks/query/builder/SourceAndTableSelector.vue @@ -62,7 +62,7 @@ const filteredSourceOptions = computed(() => { .filter((source) => source.label.toLowerCase().includes(datasourceSearchTerm.value.toLowerCase()) ) - .slice(0, 25) + .slice(0, 50) }) const tablePopover = ref(null) From 11a7304ad8c3b22ce462543f628d89f2be189b57 Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Sat, 29 Jul 2023 20:43:59 +0530 Subject: [PATCH 07/48] fix: recent query route --- frontend/src/home/HomeRecentRecords.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/home/HomeRecentRecords.vue b/frontend/src/home/HomeRecentRecords.vue index 75f340cf8..f22ee96af 100644 --- a/frontend/src/home/HomeRecentRecords.vue +++ b/frontend/src/home/HomeRecentRecords.vue @@ -30,7 +30,7 @@ function openRecord(row) { case 'Dashboard': return router.push(`/dashboard/${name}`) case 'Query': - return router.push(`/query/${name}`) + return router.push(`/query/build/${name}`) default: break } From 8d459a427c35ae9b6503b67aedf9bc646961812b Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Sat, 29 Jul 2023 20:46:45 +0530 Subject: [PATCH 08/48] fix: update sql in query --- insights/insights/doctype/insights_query/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/insights/insights/doctype/insights_query/utils.py b/insights/insights/doctype/insights_query/utils.py index 39357fb25..5361d92b0 100644 --- a/insights/insights/doctype/insights_query/utils.py +++ b/insights/insights/doctype/insights_query/utils.py @@ -87,7 +87,8 @@ class Status(Enum): def update_sql(query): - sql = InsightsDataSource.get_doc(query.data_source) + data_source = InsightsDataSource.get_doc(query.data_source) + sql = data_source.build_query(query) sql = format_query(sql) if query.sql == sql: return From 52e9f338790ba3d41bfa4960c3f1912b923101eb Mon Sep 17 00:00:00 2001 From: Saqib Ansari Date: Sun, 30 Jul 2023 22:20:58 +0530 Subject: [PATCH 09/48] refactor: flatten columns in assisted query --- .../blocks/query/builder/QueryBuilder.vue | 78 ++++++++----------- .../insights_query/insights_assisted_query.py | 5 -- .../patches/flatten_columns_in_query_json.py | 25 ++++++ insights/patches.txt | 1 + 4 files changed, 60 insertions(+), 49 deletions(-) create mode 100644 insights/insights/doctype/insights_query/patches/flatten_columns_in_query_json.py diff --git a/frontend/src/notebook/blocks/query/builder/QueryBuilder.vue b/frontend/src/notebook/blocks/query/builder/QueryBuilder.vue index 3d8a37d17..6910393f4 100644 --- a/frontend/src/notebook/blocks/query/builder/QueryBuilder.vue +++ b/frontend/src/notebook/blocks/query/builder/QueryBuilder.vue @@ -32,7 +32,7 @@ const state = computed({ const selectedTables = computed(() => { const tables = [state.value.table] state.value.joins.forEach((join) => { - join.right_table.value && tables.push(join.right_table) + join.right_table.table && tables.push(join.right_table) }) return tables }) @@ -61,9 +61,7 @@ const COLUMN = { format: {}, expression: {}, } -const GET_EMPTY_COLUMN = () => ({ - column: { ...COLUMN }, -}) +const GET_EMPTY_COLUMN = () => COLUMN function addStep(type) { if (type == 'Summarise') { @@ -114,7 +112,7 @@ const AGGREGATIONS = [ ] function setAggregation(aggregation, measure) { if (aggregation.value == 'count') { - Object.assign(measure.column, { + Object.assign(measure, { label: 'Count', column: 'count', table: 'count', @@ -123,8 +121,8 @@ function setAggregation(aggregation, measure) { aggregation: aggregation.value, }) } else { - Object.assign(measure.column, { - ...GET_EMPTY_COLUMN().column, + Object.assign(measure, { + ...GET_EMPTY_COLUMN(), aggregation: aggregation.value, }) } @@ -151,11 +149,10 @@ function isValidColumn(column) { const selectedColumns = computed(() => { const columns = [] - const addIfValid = (column) => isValidColumn(column) && columns.push(column) - state.value.columns.forEach((c) => addIfValid(c.column)) - state.value.calculations.forEach((c) => addIfValid(c.column)) - state.value.measures.forEach((c) => addIfValid(c.column)) - state.value.dimensions.forEach((c) => addIfValid(c.column)) + state.value.columns.forEach((col) => isValidColumn(col) && columns.push(col)) + state.value.calculations.forEach((col) => isValidColumn(col) && columns.push(col)) + state.value.measures.forEach((col) => isValidColumn(col) && columns.push(col)) + state.value.dimensions.forEach((col) => isValidColumn(col) && columns.push(col)) return columns.map((c) => ({ ...c, label: c.alias || c.label, description: 'local' })) }) @@ -284,7 +281,7 @@ const addStepRef = ref(null)
- +
as
@@ -293,14 +290,14 @@ const addStepRef = ref(null) > @@ -376,14 +373,14 @@ const addStepRef = ref(null) :localColumns="selectedColumns" :data_source="query.doc.data_source" :tables="selectedTables" - v-model="column.column" - @update:model-value="(c) => (column.column.alias = c.label)" + v-model="state.columns[index]" + @update:model-value="(c) => (column.alias = c.label)" /> @@ -404,28 +401,27 @@ const addStepRef = ref(null) class="flex items-center divide-x divide-gray-400 overflow-hidden rounded text-gray-800 shadow" > - +