chore(bq,sf,rs,pg): make remove drop functions instead of whole schema (

#466)

.github/workflows/snowflake-ded.yml

@@ -5,7 +5,7 @@ on:
     types: [closed, unlabeled, labeled, synchronize]
 
 env:
-  NODE_VERSION: 14
+  NODE_VERSION: 16
 
 jobs:
 

clouds/bigquery/common/run-script.js

@@ -16,11 +16,24 @@ const bar = new cliProgress.SingleBar(options, cliProgress.Presets.shades_classi
 
 const client = new BigQuery({ projectId: `${BQ_PROJECT}` });
 
+function apply_replacements (text) {
+    if (process.env.REPLACEMENTS) {
+        const replacements = process.env.REPLACEMENTS.split(' ');
+        for (let replacement of replacements) {
+            if (replacement) {
+                const pattern = new RegExp(`@@${replacement}@@`, 'g');
+                text = text.replace(pattern, process.env[replacement]);
+            }
+        }
+    }
+    return text;
+}
+
 async function runQueries (queries) {
     const n = queries.length;
     bar.start(n, 0);
     for (let i = 0; i < n; i++) {
-        let query = queries[i];
+        let query = apply_replacements(queries[i]);
 
         const pattern = /(FUNCTION|PROCEDURE)\s+(.*?)[(\n]/g;
         const results = pattern.exec(query);

clouds/bigquery/modules/Makefile

@@ -97,7 +97,9 @@ test: check $(NODE_MODULES_DEV)
 
 remove: check
 	echo "Removing modules..."
-	$(BQ) rm -r -f -d $(BQ_DEPLOY_DATASET)
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
+	$(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
 clean:
 	echo "Cleaning modules..."

clouds/bigquery/modules/sql/constructors/ST_TILEENVELOPE.sql

@@ -6,8 +6,8 @@ CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.ST_TILEENVELOPE`
 (zoomLevel INT64, xTile INT64, yTile INT64)
 RETURNS GEOGRAPHY
 AS (
-    `@@BQ_DATASET@@.QUADINT_BOUNDARY`(
-        `@@BQ_DATASET@@.QUADINT_FROMZXY`(
+    `@@BQ_DATASET@@.QUADBIN_BOUNDARY`(
+        `@@BQ_DATASET@@.QUADBIN_FROMZXY`(
             zoomlevel, xtile, ytile
         )
     )

clouds/bigquery/modules/test/constructors/ST_TILEENVELOPE.test.js

@@ -15,7 +15,9 @@ test('ST_TILEENVELOPE should work', async () => {
 
 test('ST_TILEENVELOPE should fail if any NULL argument', async () => {
     const query = `
-        SELECT \`@@BQ_DATASET@@.ST_TILEENVELOPE\`(10, 384, null)
+        SELECT \`@@BQ_DATASET@@.ST_TILEENVELOPE\`(10, 384, null) AS geog
     `;
-    await expect(runQuery(query)).rejects.toThrow();
+    const rows = await runQuery(query);
+    expect(rows.length).toEqual(1);
+    expect(rows[0].geog).toEqual(null);
 });

clouds/postgres/common/run_script.py

@@ -9,6 +9,16 @@
 function = ''
 
 
+def apply_replacements(text):
+    if os.environ.get('REPLACEMENTS'):
+        replacements = os.environ.get('REPLACEMENTS').split(' ')
+        for replacement in replacements:
+            if replacement:
+                pattern = re.compile(f'@@{replacement}@@', re.MULTILINE)
+                text = pattern.sub(os.environ.get(replacement, ''), text)
+    return text
+
+
 def run_queries(queries):
     global function
     with connect(
@@ -20,7 +30,7 @@ def run_queries(queries):
         conn.autocommit = True
         with conn.cursor() as cursor:
             for i in trange(len(queries), ncols=97):
-                query = queries[i]
+                query = apply_replacements(queries[i])
                 pattern = os.environ['PG_SCHEMA'] + '.(.*?)[(|\n]'
                 result = re.search(pattern, query)
                 if result:

clouds/postgres/modules/Makefile

@@ -61,7 +61,8 @@ test: check venv3 $(NODE_MODULES_DEV)
 
 remove: venv3
 	echo "Removing modules..."
-	$(VENV3_BIN)/python $(COMMON_DIR)/run_query.py "DROP SCHEMA IF EXISTS $(PG_SCHEMA) CASCADE;"
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	$(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
 clean:
 	echo "Cleaning modules..."

clouds/redshift/common/run_script.py

@@ -10,6 +10,16 @@
 function = ''
 
 
+def apply_replacements(text):
+    if os.environ.get('REPLACEMENTS'):
+        replacements = os.environ.get('REPLACEMENTS').split(' ')
+        for replacement in replacements:
+            if replacement:
+                pattern = re.compile(f'@@{replacement}@@', re.MULTILINE)
+                text = pattern.sub(os.environ.get(replacement, ''), text)
+    return text
+
+
 def run_queries(queries):
     global function
     with connect(
@@ -22,7 +32,7 @@ def run_queries(queries):
         filter = os.environ.get('FILTER')
         with conn.cursor() as cursor:
             for i in trange(len(queries) if not filter else 1, ncols=97):
-                query = queries[i]
+                query = apply_replacements(queries[i])
                 if (not filter) or (filter in query):
                     pattern = os.environ['RS_SCHEMA'] + '.(.*?)[(|\n]'
                     result = re.search(pattern, str(query))

clouds/redshift/modules/Makefile

@@ -61,7 +61,8 @@ test: check venv3 $(NODE_MODULES_DEV)
 
 remove: venv3
 	echo "Removing modules..."
-	$(VENV3_BIN)/python $(COMMON_DIR)/run_query.py "DROP SCHEMA IF EXISTS $(RS_SCHEMA) CASCADE;"
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	$(VENV3_BIN)/python $(COMMON_DIR)/run_script.py $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
 clean:
 	echo "Cleaning modules..."

clouds/snowflake/common/run-script.js

@@ -42,11 +42,24 @@ async function runQuery (query) {
     });
 }
 
+function apply_replacements (text) {
+    if (process.env.REPLACEMENTS) {
+        const replacements = process.env.REPLACEMENTS.split(' ');
+        for (let replacement of replacements) {
+            if (replacement) {
+                const pattern = new RegExp(`@@${replacement}@@`, 'g');
+                text = text.replace(pattern, process.env[replacement]);
+            }
+        }
+    }
+    return text;
+}
+
 async function runQueries (queries) {
     const n = queries.length;
     bar.start(n, 0);
     for (let i = 0; i < n; i++) {
-        let query = `BEGIN\n${queries[i]}\nEND;`;
+        let query = apply_replacements (`BEGIN\n${queries[i]}\nEND;`);
 
         const pattern = /(FUNCTION|PROCEDURE)\s+(.*?)[(\n]/g;
         const results = pattern.exec(query);

clouds/snowflake/modules/Makefile

@@ -106,7 +106,8 @@ test: check $(NODE_MODULES_DEV)
 
 remove: $(NODE_MODULES_DEV)
 	echo "Removing modules..."
-	$(COMMON_DIR)/run-query.js "DROP SCHEMA IF EXISTS $(SF_SCHEMA) CASCADE;"
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	$(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
 remove-share: $(NODE_MODULES_DEV)
 	echo "Removing share..."

clouds/snowflake/modules/test/constructors/ST_TILEENVELOPE.test.js

@@ -13,9 +13,11 @@ test('ST_TILEENVELOPE should work', async () => {
     expect(JSON.stringify(rows[0].GEOG3)).toEqual('{"coordinates":[[[-44.99998927116395,45.000002199069606],[-44.99998927116395,44.99999461263666],[-45,44.99999461263666],[-45,45.000002199069606],[-44.99998927116395,45.000002199069606]]],"type":"Polygon"}');
 });
 
-test('ST_TILEENVELOPE should fail if any NULL argument', async () => {
+test('ST_TILEENVELOPE should return NULL if any NULL argument', async () => {
     const query = `
-        SELECT ST_TILEENVELOPE(10, 384, null)
+        SELECT ST_TILEENVELOPE(10, 384, null) as geog
     `;
-    await expect(runQuery(query)).rejects.toThrow();
+    const rows = await runQuery(query);
+    expect(rows.length).toEqual(1);
+    expect(rows[0].GEOG).toEqual(null);
 });