Skip to content

Commit

Permalink
feat(sf): refactor at snowflake native app to an installer (#512)
Browse files Browse the repository at this point in the history
  • Loading branch information
@vdelacruzb
vdelacruzb authored May 31, 2024
1 parent 2fc74a5 commit c9d4ef9
Show file tree
Hide file tree
Showing 8 changed files with 156 additions and 208 deletions.
2 changes: 2 additions & 0 deletions clouds/snowflake/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ build-native-app-setup-script:
rm -rf $(BUILD_DIR)
$(MAKE) build-libraries
mkdir -p $(BUILD_DIR)
$(MAKE) -C modules build SF_SCHEMA=@@SF_SCHEMA@@
cp modules/build/modules.sql $(BUILD_DIR)
$(MAKE) -C modules build-native-app-setup-script
cp modules/build/setup_script.sql $(BUILD_DIR)

Expand Down
199 changes: 9 additions & 190 deletions clouds/snowflake/common/build_native_app_setup_script.js
View file
Original file line number Diff line number Diff line change
@@ -1,143 +1,16 @@
#!/usr/bin/env node

// Build the setup_script for the native app file based on the input filters
// and ordered to solve the dependencies
// Build the setup_script for the native app installer setup file

// ./build_native_app_setup_script.js modules --output=build --diff="clouds/snowflake/modules/sql/quadbin/QUADBIN_TOZXY.sql"
// ./build_native_app_setup_script.js modules --output=build --functions=ST_TILEENVELOPE
// ./build_native_app_setup_script.js modules --output=build --modules=quadbin
// ./build_native_app_setup_script.js modules --output=build --production --dropfirst
// ./build_native_app_setup_script.js modules --output=build --libs_build_dir=../libraries/javascript/build native_app_dir=../native_app --production --dropfirst

const fs = require('fs');
const path = require('path');
const argv = require('minimist')(process.argv.slice(2));

const inputDirs = argv._[0] && argv._[0].split(',');
const outputDir = argv.output || 'build';
const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build';
const nativeAppDir = argv.native_app_dir || '../native_app';
const diff = argv.diff || [];
const nodeps = argv.nodeps;
let modulesFilter = (argv.modules && argv.modules.split(',')) || [];
let functionsFilter = (argv.functions && argv.functions.split(',')) || [];
let all = !(diff.length || modulesFilter.length || functionsFilter.length);

if (all) {
console.log('- Build all');
} else if (diff && diff.length) {
console.log(`- Build input diff: ${argv.diff}`);
} else if (modulesFilter && modulesFilter.length) {
console.log(`- Build input modules: ${argv.modules}`);
} else if (functionsFilter && functionsFilter.length) {
console.log(`- Build input functions: ${argv.functions}`);
}

// Convert diff to modules
if (diff.length) {
const patternsAll = [
/\.github\/workflows\/snowflake\.yml/,
/clouds\/snowflake\/common\/.+/,
/clouds\/snowflake\/libraries\/.+/,
/clouds\/snowflake\/.*Makefile/,
/clouds\/snowflake\/version/
];
const patternModulesSql = /clouds\/snowflake\/modules\/sql\/([^\s]*?)\//g;
const patternModulesTest = /clouds\/snowflake\/modules\/test\/([^\s]*?)\//g;
const diffAll = patternsAll.some(p => diff.match(p));
if (diffAll) {
console.log('-- all');
all = diffAll;
} else {
const modulesSql = [...diff.matchAll(patternModulesSql)].map(m => m[1]);
const modulesTest = [...diff.matchAll(patternModulesTest)].map(m => m[1]);
const diffModulesFilter = [...new Set(modulesSql.concat(modulesTest))];
if (diffModulesFilter) {
console.log(`-- modules: ${diffModulesFilter}`);
modulesFilter = diffModulesFilter;
}
}
}

// Extract functions
const functions = [];
for (let inputDir of inputDirs) {
const sqldir = path.join(inputDir, 'sql');
const modules = fs.readdirSync(sqldir);
modules.forEach(module => {
const moduledir = path.join(sqldir, module);
if (fs.statSync(moduledir).isDirectory()) {
const files = fs.readdirSync(moduledir);
files.forEach(file => {
if (file.endsWith('.sql')) {
const name = path.parse(file).name;
const content = fs.readFileSync(path.join(moduledir, file)).toString().replace(/--.*\n/g, '');
functions.push({
name,
module,
content,
dependencies: []
});
}
});
}
});
}

// Check filters
modulesFilter.forEach(m => {
if (!functions.map(fn => fn.module).includes(m)) {
console.log(`ERROR: Module not found ${m}`);
process.exit(1);
}
});
functionsFilter.forEach(f => {
if (!functions.map(fn => fn.name).includes(f)) {
console.log(`ERROR: Function not found ${f}`);
process.exit(1);
}
});

// Extract function dependencies
if (!nodeps) {
functions.forEach(mainFunction => {
functions.forEach(depFunction => {
if (mainFunction.name != depFunction.name) {
if (mainFunction.content.includes(`SCHEMA@@.${depFunction.name}(`)) {
mainFunction.dependencies.push(depFunction.name);
}
}
});
});
}

// Check circular dependencies
if (!nodeps) {
functions.forEach(mainFunction => {
functions.forEach(depFunction => {
if (mainFunction.dependencies.includes(depFunction.name) &&
depFunction.dependencies.includes(mainFunction.name)) {
console.log(`ERROR: Circular dependency between ${mainFunction.name} and ${depFunction.name}`);
process.exit(1);
}
});
});
}

// Filter and order functions
const output = [];
function add (f, include) {
include = include || all || functionsFilter.includes(f.name) || modulesFilter.includes(f.module);
for (const dependency of f.dependencies) {
add(functions.find(f => f.name === dependency), include);
}
if (!output.map(f => f.name).includes(f.name) && include) {
output.push({
name: f.name,
content: f.content
});
}
}
functions.forEach(f => add(f));

// Replace environment variables
let separator;
Expand All @@ -146,7 +19,7 @@ if (argv.production) {
} else {
separator = '\n-->\n'; // marker to future SQL split
}
let content = output.map(f => fetchPermissionsGrant(f.content)).join(separator);
let content = fs.readFileSync(path.resolve(nativeAppDir, 'SETUP_SCRIPT.sql')).toString();

function apply_replacements (text) {
const libraries = [... new Set(text.match(new RegExp('@@SF_LIBRARY_.*@@', 'g')))];
Expand All @@ -172,80 +45,26 @@ function apply_replacements (text) {
return text;
}

function getFunctionSignatures (functionMatches)
{
const functSignatures = []
for (const functionMatch of functionMatches) {
//Remove spaces and diacritics
let qualifiedFunctName = functionMatch[0].split('(')[0].replace(/\s+/gm,'');
qualifiedFunctNameArr = qualifiedFunctName.split('.');
const functName = qualifiedFunctNameArr[qualifiedFunctNameArr.length - 1];
if (functName.startsWith('_'))
{
continue;
}
//Remove diacritics and go greedy to take the outer parentheses
let functArgs = functionMatch[0].matchAll(new RegExp('(?<=\\()(.*)(?=\\))','g')).next().value;
if (functArgs)
{
functArgs = functArgs[0];
}
else
{
continue;
}
functArgs = functArgs.split(',')
let functArgsTypes = [];
for (const functArg of functArgs) {
const functArgSplitted = functArg.trim(' ').split(' ');
functArgsTypes.push(functArgSplitted[functArgSplitted.length - 1]);
}
const functSignature = qualifiedFunctName + '(' + functArgsTypes.join(',') + ')';
functSignatures.push(functSignature)
}
return functSignatures
}

function fetchPermissionsGrant (content)
{
let fileContent = content.split('\n');
for (let i = 0 ; i < fileContent.length; i++)
{
if (fileContent[i].startsWith('--'))
{
delete fileContent[i];
}
}
fileContent = fileContent.join(' ').replace(/[\p{Diacritic}]/gu, '').replace(/\s+/gm,' ');
const functionMatches = fileContent.matchAll(new RegExp(/(?<=(?<!TEMP )FUNCTION)(.*?)(?=RETURNS)/gs));
const functSignatures = getFunctionSignatures(functionMatches).map(f => `GRANT USAGE ON FUNCTION ${f} TO APPLICATION ROLE @@APP_ROLE@@;`).join('\n')
const procMatches = fileContent.matchAll(new RegExp(/(?<=PROCEDURE)(.*?)(?=AS)/gs));
const procSignatures = getFunctionSignatures(procMatches).map(f => `GRANT USAGE ON PROCEDURE ${f} TO APPLICATION ROLE @@APP_ROLE@@;`).join('\n')
return content + functSignatures +procSignatures
}

if (argv.dropfirst) {
const header = fs.readFileSync(path.resolve(__dirname, 'DROP_FUNCTIONS.sql')).toString();
let header = fs.readFileSync(path.resolve(__dirname, 'DROP_FUNCTIONS.sql')).toString();
const pattern = new RegExp('@@SF_SCHEMA@@', 'g');
header = header.replace(pattern, '@@SF_APP_SCHEMA@@');
content = header + separator + content
}

const header = `CREATE OR REPLACE APPLICATION ROLE @@APP_ROLE@@;
CREATE OR ALTER VERSIONED SCHEMA @@SF_SCHEMA@@;
GRANT USAGE ON SCHEMA @@SF_SCHEMA@@ TO APPLICATION ROLE @@APP_ROLE@@;\n`;
CREATE OR ALTER VERSIONED SCHEMA @@SF_APP_SCHEMA@@;
GRANT USAGE ON SCHEMA @@SF_APP_SCHEMA@@ TO APPLICATION ROLE @@APP_ROLE@@;\n`;

let additionalTables = '';
if (argv.production) {
additionalTables = fs.readFileSync(path.resolve(nativeAppDir, 'ADDITIONAL_TABLES.sql')).toString() + separator;
}

const footer = fetchPermissionsGrant (fs.readFileSync(path.resolve(__dirname, 'VERSION.sql')).toString());
content = header + separator + additionalTables + content + separator + footer;
content = header + separator + additionalTables + content;

content = apply_replacements(content);

// Execute as caller replacement
content = content.replace(/EXECUTE\s+AS\s+CALLER/g, 'EXECUTE AS OWNER');

// Write setup_script.sql file
fs.writeFileSync(path.join(outputDir, 'setup_script.sql'), content);
console.log(`Write ${outputDir}/setup_script.sql`);
9 changes: 5 additions & 4 deletions clouds/snowflake/modules/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ BAIL=--bail
endif

REPLACEMENTS = "SF_SCHEMA SF_VERSION_FUNCTION SF_PACKAGE_VERSION SF_SHARE APP_ROLE"
NATIVE_APP_REPLACEMENTS = "SF_APP_SCHEMA APP_ROLE"

include $(COMMON_DIR)/Makefile

Expand Down Expand Up @@ -73,11 +74,11 @@ build-native-app-setup-script: $(NODE_MODULES_DEV)
echo "Building native app setup script..."
rm -rf $(BUILD_DIR)
mkdir $(BUILD_DIR)
SF_SCHEMA=$(SF_UNQUALIFIED_SCHEMA) APP_ROLE=app_public \
REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
SF_APP_SCHEMA=$(SF_UNQUALIFIED_SCHEMA) APP_ROLE=app_public \
REPLACEMENTS=$(NATIVE_APP_REPLACEMENTS) \
$(COMMON_DIR)/build_native_app_setup_script.js $(MODULES_DIRS) \
--output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --native_app_dir=$(NATIVE_APP_DIR) --diff="$(diff)" \
--modules=$(modules) --functions=$(functions) --production=$(production) --nodeps=$(nodeps) --dropfirst=1
--output=$(BUILD_DIR) --libs_build_dir=$(LIBS_BUILD_DIR) --native_app_dir=$(NATIVE_APP_DIR) \
--production=$(production) --dropfirst=1

deploy: check build
echo "Deploying modules..."
Expand Down
3 changes: 3 additions & 0 deletions clouds/snowflake/native_app/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@ help:
build:
rm -rf $(DIST_DIR)
mkdir -p $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)
cp $(BUILD_DIR)/modules.sql $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/
cp $(BUILD_DIR)/setup_script.sql $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/
sed 's/@@VERSION@@/$(APP_VERSION)/g' $(APP_DIR)/manifest.yml > $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/manifest.yml
cp $(APP_DIR)/README.md $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/
cp $(APP_DIR)/get_modules_sql_from_stage.py $(DIST_DIR)/$(APP_PACKAGE_DIST_NAME)/


deploy-app-package:
Expand Down Expand Up @@ -92,6 +94,7 @@ deploy-app:
else \
echo "Upgrading native app... (this may take a while)"; \
$(COMMON_DIR)/run-query.js "ALTER APPLICATION $(APP_NAME) UPGRADE;"; \
$(MAKE) extra-app-deploy; \
fi

extra-app-deploy::
Expand Down
49 changes: 36 additions & 13 deletions clouds/snowflake/native_app/README.md
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,50 @@
## ANALYTICS TOOLBOX CORE
## ANALYTICS TOOLBOX CORE INSTALLER

### Introduction

The CARTO Analytics Toolbox for Snowflake is composed of a set of user-defined functions and procedures organized in a set of modules based on the functionality they offer. This app gives you access to the Open Source modules included in the CARTO's Analytics Toolbox, supporting different spatial indexes and other geospatial operations: quadkeys, H3, S2, placekey, geometry constructors, accessors, transformations, etc.

We recommend that at the moment of installing the app you name the app "CARTO". The next guidelines and examples will assume that in order to simplify the onboarding process.

### Installation

#### Grant Usage
#### Install the Analytics Toolbox

This step is required by procedures that have input tables/queries or output tables. The user will have to manually provide permissions to access the tables of a given database. Providing ALL permissions on an SCHEMA ensures that the app can write new tables in that schema.
This Native App is an installer so it does not contain the actual Analytics Toolbox functions and procedures. For the sake of documenting the process, we'll will assume a database named CARTO, as well as a schema named CARTO in that database, also we assume the app to be called CARTO_INSTALLER. The next guidelines and examples will assume that in order to simplify the onboarding process.

On the other hand, those tables generated by a procedure belong to the app itself. If the user wants to recover control over those tables (performing SELECT, UPDATE, DELETE...), updating the ownership on those tables is required.
All the database, schema and user can have a different name, but remember to adapt the code snippets accordingly.

```
-- Set read and write permissions
GRANT USAGE ON DATABASE <database> TO APPLICATION CARTO;
GRANT ALL ON SCHEMA <database>.<schema> TO APPLICATION CARTO;
GRANT SELECT ON TABLE IN SCHEMA <database>.<schema>.<table> TO APPLICATION CARTO;
-- Set admin permissions
USE ROLE ACCOUNTADMIN;
-- Create the carto database
CREATE DATABASE CARTO;
-- Create the carto schema
CREATE SCHEMA CARTO.CARTO;
-- Grant all to sysadmin role
GRANT ALL ON SCHEMA CARTO.CARTO TO ROLE SYSADMIN;
-- Set create function and procedure permissions
GRANT USAGE ON DATABASE CARTO TO APPLICATION CARTO;
GRANT USAGE, CREATE FUNCTION, CREATE PROCEDURE ON SCHEMA CARTO.CARTO TO APPLICATION CARTO;
-- Generate the installer procedure in the specified location
CALL CARTO_INSTALLER.CARTO.GENERATE_INSTALLER('CARTO.CARTO');
-- Update ownership of the install procedure
GRANT OWNERSHIP ON PROCEDURE CARTO.CARTO.INSTALL(STRING, STRING) TO ROLE ACCOUNTADMIN REVOKE CURRENT GRANTS;
-- Grant usage to public role
GRANT USAGE ON DATABASE CARTO TO ROLE PUBLIC;
GRANT USAGE ON SCHEMA CARTO.CARTO TO ROLE PUBLIC;
GRANT SELECT ON FUTURE TABLES IN SCHEMA CARTO.CARTO TO ROLE PUBLIC;
GRANT SELECT ON FUTURE VIEWS IN SCHEMA CARTO.CARTO TO ROLE PUBLIC;
GRANT USAGE ON FUTURE FUNCTIONS IN SCHEMA CARTO.CARTO TO ROLE PUBLIC;
GRANT USAGE ON FUTURE PROCEDURES IN SCHEMA CARTO.CARTO TO ROLE PUBLIC;
-- Update ownership (when a table is created within the app)
GRANT OWNERSHIP ON TABLE <database>.<schema>.<output_table> TO ROLE ACCOUNTADMIN;
-- Install the Analytics Toolbox in CARTO.CARTO
CALL CARTO.CARTO.INSTALL('CARTO_INSTALLER', 'CARTO.CARTO');
```

### Usage Examples
Expand All @@ -33,7 +56,7 @@ Please refer to CARTO's [SQL reference](https://docs.carto.com/data-and-analysis
Returns an array with all the H3 cell indexes **with centers** contained in a given polygon.

```
SELECT carto.H3_POLYFILL(
SELECT CARTO.CARTO.H3_POLYFILL(
TO_GEOGRAPHY('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))'), 4);
-- 842da29ffffffff
-- 843f725ffffffff
Expand Down
Loading

0 comments on commit c9d4ef9

Please sign in to comment.