Skip to content

Commit

Permalink
Merge branch 'main' into pp_sql
Browse files Browse the repository at this point in the history
  • Loading branch information
@takegue
takegue committed Nov 12, 2023
2 parents ca349da + 4c715c2 commit c11f1ea
Show file tree
Hide file tree
Showing 2 changed files with 274 additions and 233 deletions.
357 changes: 173 additions & 184 deletions bigquery/@default/v0/@routines/zgensql__staleness_check/test.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,195 +32,184 @@ begin
as select date '2006-01-02' as date_jst
;

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, []
, [("__ANY__", ["__ANY__"])]
, to_json(struct(interval 0 hour as tolerate_staleness, @@project_id as default_project_id))
) into ret
;

assert ret[safe_offset(0)] is not null
as "Empty sourcs will be always staleness";

-- INFORMATIN_SCHEMA sourcs
execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2.INFORMATION_SCHEMA", "PARTITIONS")]
, [("__ANY__", ["__ANY__"])]
, to_json(struct(interval 0 hour as tolerate_staleness, @@project_id as default_project_id))
) into ret
;

assert ret[safe_offset(0)] is not null
as "INFORMATIN_SCHEMA sourcs will get always staled ";

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2", "ref1")]
, [("20060102", ["20060102"])]
, to_json(struct(interval 0 hour as tolerate_staleness, @@project_id as default_project_id))
) into ret
;

assert ret[safe_offset(0)] is null
as "Not stale partition: dest1 > ref1"
;

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2", "ref1")]
, [("20060102", ["20060102"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
, current_timestamp() as force_expired_at
))
) into ret
;

assert ret[safe_offset(0)] = '20060102'
as "Stale partition: force_expired_at option"
;

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [
(string(null), "zpreview_test2", "ref1")
, (string(null), "zpreview_test2", "ref_20060103")
]
, [("20060102", ["20060102"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
) into ret
;

assert ret[safe_offset(0)] is null
as "Not stale partition: Inconsistent sources"
;

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [
(string(null), "zpreview_test2", "ref1")
, (string(null), "zpreview_test2", "ref_20060103")
]
, [("20060102", ["__ANY__"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
) into ret
;

assert ret[safe_offset(0)] = '20060102'
as "Satle partition: __ANY__"
;

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [
(string(null), "zpreview_test2", "ref1")
, (string(null), "zpreview_test2", "ref_20060102")
execute immediate (
select as value
"with "
|| string_agg(
format('cte%d as (%s) ', ix, sql)
, ',\n'
)
|| format("""
select array<struct<
expected array<string>
, actual array<string>>
>[
%s
]
, [("20060102", ["20060102"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
into ret
;

assert ret[safe_offset(0)] = '20060102'
as "Stale partition under some source's partition is fresher than destination: ref_20060102 > dest1 > ref1"
;

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2", "ref_no_partition")]
, [('20060102', ["__NULL__"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
into ret
;


assert ret[safe_offset(0)] = '20060102'
as "Stale partition under non-partitioned source: dest1 > ref_no_partition"
;

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2", "ref_no_partition")]
, [('20060102', ["20060102"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
into ret
;

assert ret[safe_offset(0)] = '20060102'
as "Stale partition between alignment between Non-partition table and partition table"
;


execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest_no_partition")
, [(string(null), "zpreview_test2", "ref_no_partition")]
, [('__NULL__', ["__NULL__"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
"""
, string_agg(
format('(select as value struct(cast(%T as array<string>) as expected, cast(cte%d as struct<ret array<string>>).ret as actual) from cte%d)', expected, ix, ix)
, ',\n'
))
)
into ret
;

assert ret[safe_offset(0)] = '__NULL__'
as "Stale non-partitioned table under non-partitioned source: dest_no_partition > ref_no_partition"
;

execute immediate
`v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest_no_partition")
, [
(string(null), "zpreview_test2", "ref_no_partition")
, (string(null), "zpreview_test2.INFORMATION_SCHEMA", "VIEWS")
, (string(null), "zpreview_test2.INFORMATION_SCHEMA", "PARTITIONS")
, (string(null), "zpreview_test2.INFORMATION_SCHEMA", "TABLES")
from unnest(
array<struct< msg string, expected array<string>,sql string>>[
(
"Empty sourcs will be always staleness"
, ['__ANY__']
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, []
, [("__ANY__", ["__ANY__"])]
, to_json(struct(interval 0 hour as tolerate_staleness, @@project_id as default_project_id))
)
)
, (
"INFORMATIN_SCHEMA sourcs will get always staled"
, ['__ANY__']
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2.INFORMATION_SCHEMA", "PARTITIONS")]
, [("__ANY__", ["__ANY__"])]
, to_json(struct(interval 0 hour as tolerate_staleness, @@project_id as default_project_id))
)
)
, (
"Not stale partition: dest1 > ref1"
, []
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2", "ref1")]
, [("20060102", ["20060102"])]
, to_json(struct(interval 0 hour as tolerate_staleness, @@project_id as default_project_id))
)
)
, (
"Stale partition: force_expired_at option"
, ["20060102"]
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2", "ref1")]
, [("20060102", ["20060102"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
, current_timestamp() as force_expired_at
))
)
)
, (
"Stale partition: Inconsistent sources"
, ["20060102"]
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [
(string(null), "zpreview_test2", "ref1")
, (string(null), "zpreview_test2", "ref_20060103")
]
, [("20060102", ["20060102"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
)
, (
"Satle partition: __ANY__"
, []
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [
(string(null), "zpreview_test2", "ref1")
, (string(null), "zpreview_test2", "ref_20060103")
]
, [("20060102", ["__ANY__"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
)
, (
"Stale partition under some source's partition is fresher than destination: ref_20060102 > dest1 > ref1"
, ['20060102']
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [
(string(null), "zpreview_test2", "ref1")
, (string(null), "zpreview_test2", "ref_20060102")
]
, [("20060102", ["20060102"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
)
, (
"Stale partition under non-partitioned source: dest1 > ref_no_partition"
, ["20060102"]
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2", "ref_no_partition")]
, [('20060102', ["__NULL__"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
)
, (
"Stale partition between alignment between Non-partition table and partition table"
, ['20060102']
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest1")
, [(string(null), "zpreview_test2", "ref_no_partition")]
, [('20060102', ["20060102"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
)
, (
"Stale non-partitioned table under non-partitioned source: dest_no_partition > ref_no_partition"
, ['__NULL__']
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest_no_partition")
, [(string(null), "zpreview_test2", "ref_no_partition")]
, [('__NULL__', ["__NULL__"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
)
, (
"Stale non-partitioned table under non-partitioned source: dest_no_partition > ref_no_partition"
, ['__NULL__']
, `v0.zgensql__staleness_check`(
(null, "zpreview_test2", "dest_no_partition")
, [
(string(null), "zpreview_test2", "ref_no_partition")
, (string(null), "zpreview_test2.INFORMATION_SCHEMA", "VIEWS")
, (string(null), "zpreview_test2.INFORMATION_SCHEMA", "PARTITIONS")
, (string(null), "zpreview_test2.INFORMATION_SCHEMA", "TABLES")
]
, [('__NULL__', ["__NULL__"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
)
]
, [('__NULL__', ["__NULL__"])]
, to_json(struct(
interval 0 hour as tolerate_staleness
, @@project_id as default_project_id
))
)
into ret
;
assert ret[safe_offset(0)] = '__NULL__'
as "Stale non-partitioned table under non-partitioned source: dest_no_partition > ref_no_partition"
) with offset ix
)
;

drop schema if exists `zpreview_test2` cascade;

-- drop schema if exists `zpreview_test2` cascade;
exception when error then
drop schema if exists `zpreview_test2` cascade;
-- drop schema if exists `zpreview_test2` cascade;
raise using message = @@error.message;
end
Loading

0 comments on commit c11f1ea

Please sign in to comment.