diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs index b213734b68..5badccba7c 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/ExternalCdpDataSource.cs @@ -36,6 +36,8 @@ public ExternalCdpDataSource(DName name, string datasetName, ServiceCapabilities public TabularDataQueryOptions QueryOptions => new TabularDataQueryOptions(this); + public bool HasCachedCountRows => false; + public string Name => EntityName.Value; public bool IsSelectable => ServiceCapabilities.IsSelectable; diff --git a/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs b/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs index 1c30c12158..7d69e78e0c 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Entities/External/IExternalTabularDataSource.cs @@ -11,6 +11,13 @@ internal interface IExternalTabularDataSource : IExternalDataSource, IDisplayMap { TabularDataQueryOptions QueryOptions { get; } + /// + /// Some data sources (like Dataverse) may return a cached value for + /// the number of rows (calls to CountRows) instead of always retrieving + /// the latest count. + /// + bool HasCachedCountRows { get; } + IReadOnlyList GetKeyColumns(); IEnumerable GetKeyColumns(IExpandInfo expandInfo); @@ -23,4 +30,4 @@ internal interface IExternalTabularDataSource : IExternalDataSource, IDisplayMap bool CanIncludeExpand(IExpandInfo parentExpandInfo, IExpandInfo expandToAdd); } -} \ No newline at end of file +} diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index 68008bb151..288586637b 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -745,6 +745,8 @@ internal static class TexlStrings public static ErrorResourceKey WarnDeferredType = new ErrorResourceKey("WarnDeferredType"); public static ErrorResourceKey ErrColRenamedTwice_Name = new ErrorResourceKey("ErrColRenamedTwice_Name"); + public static ErrorResourceKey WrnCountRowsMayReturnCachedValue = new ErrorResourceKey("WrnCountRowsMayReturnCachedValue"); + public static StringGetter InfoMessage = (b) => StringResources.Get("InfoMessage", b); public static StringGetter InfoNode_Node = (b) => StringResources.Get("InfoNode_Node", b); public static StringGetter InfoTok_Tok = (b) => StringResources.Get("InfoTok_Tok", b); diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/CountRows.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/CountRows.cs index e04184c204..60caada7f3 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/CountRows.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/CountRows.cs @@ -5,6 +5,7 @@ using System.Linq; using Microsoft.PowerFx.Core.App.ErrorContainers; using Microsoft.PowerFx.Core.Binding; +using Microsoft.PowerFx.Core.Binding.BindInfo; using Microsoft.PowerFx.Core.Entities; using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Functions; @@ -68,6 +69,32 @@ public override bool IsServerDelegatable(CallNode callNode, TexlBinding binding) return TryGetValidDataSourceForDelegation(callNode, binding, out var dataSource, out var preferredFunctionDelegationCapability); } + public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors) + { + base.CheckSemantics(binding, args, argTypes, errors); + if (args[0] is not FirstNameNode node) + { + // No additional check + return; + } + + var info = binding.GetInfo(args[0] as FirstNameNode); + if (info.Kind != BindKind.Data) + { + // No additional check + return; + } + + if (argTypes[0].AssociatedDataSources?.Count == 1) + { + var associatedDataSource = argTypes[0].AssociatedDataSources.Single(); + if (associatedDataSource.HasCachedCountRows) + { + errors.EnsureError(DocumentErrorSeverity.Warning, node, TexlStrings.WrnCountRowsMayReturnCachedValue); + } + } + } + // See if CountDistinct delegation is available. If true, we can make use of it on primary key as a workaround for CountRows delegation internal bool TryGetValidDataSourceForDelegation(CallNode callNode, TexlBinding binding, out IExternalDataSource dataSource, out DelegationCapability preferredFunctionDelegationCapability) { diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index 3f818dcd1a..e063733cf3 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4332,6 +4332,10 @@ Can't delegate {0}: contains a behavior function '{1}'. Warning message. + + CountRows may return a cached value. Use CountIf(DataSource, true) to get the latest count. + {Locked=CountRows}. Warning message when an expression with the CountRows function is used with a data source that caches its size. + Determines if the supplied text has a match of the supplied text format. Description of 'IsMatch' function. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs index 7905edcdd9..177c71f5a3 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDVEntity.cs @@ -15,6 +15,13 @@ namespace Microsoft.PowerFx.Core.Tests.AssociatedDataSourcesTests { public class AccountsEntity : IExternalEntity, IExternalDataSource { + private readonly bool _hasCachedCountRows; + + public AccountsEntity(bool hasCachedCountRows = false) + { + this._hasCachedCountRows = hasCachedCountRows; + } + public DName EntityName => new DName("Accounts"); public string Name => "Accounts"; @@ -33,7 +40,7 @@ public class AccountsEntity : IExternalEntity, IExternalDataSource public bool IsClearable => true; - DType IExternalEntity.Type => AccountsTypeHelper.GetDType(); + DType IExternalEntity.Type => AccountsTypeHelper.GetDType(this._hasCachedCountRows); IExternalDataEntityMetadataProvider IExternalDataSource.DataEntityMetadataProvider => throw new NotImplementedException(); @@ -54,14 +61,15 @@ internal static class AccountsTypeHelper "name`Account Name`:s, numberofemployees:n, primarytwitterid:s, stockexchange:s, telephone1:s, telephone2:s, telephone3:s, tickersymbol:s, versionnumber:n, " + "websiteurl:h, nonsearchablestringcol`Non-searchable string column`:s, nonsortablestringcolumn`Non-sortable string column`:s]"; - public static DType GetDType() + public static DType GetDType(bool hasCachedCountRows = false) { DType accountsType = TestUtils.DT2(SimplifiedAccountsSchema); var dataSource = new TestDataSource( "Accounts", accountsType, keyColumns: new[] { "accountid" }, - selectableColumns: new[] { "name", "address1_city", "accountid", "address1_country", "address1_line1" }); + selectableColumns: new[] { "name", "address1_city", "accountid", "address1_country", "address1_line1" }, + hasCachedCountRows: hasCachedCountRows); var displayNameMapping = dataSource.DisplayNameMapping; displayNameMapping.Add("name", "Account Name"); displayNameMapping.Add("address1_city", "Address 1: City"); diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDelegationValidation.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDelegationValidation.cs index b8c5761a88..50ff69317a 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDelegationValidation.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/AssociatedDataSourcesTests/TestDelegationValidation.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Text; using Microsoft.PowerFx.Core.Entities.QueryOptions; +using Microsoft.PowerFx.Core.Errors; using Microsoft.PowerFx.Core.Tests.Helpers; using Microsoft.PowerFx.Core.Texl; using Microsoft.PowerFx.Types; @@ -100,5 +101,38 @@ private void TestDelegableExpressions(Features features, string expression, bool // validate we can generate the display expression string displayExpr = engine.GetDisplayExpression(expression, symbolTable); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void TestCountRowsWarningForCachedData(bool isCachedData) + { + var symbolTable = new DelegatableSymbolTable(); + symbolTable.AddEntity(new AccountsEntity(isCachedData)); + var config = new PowerFxConfig(Features.PowerFxV1) + { + SymbolTable = symbolTable + }; + + var engine = new Engine(config); + var result = engine.Check("CountRows(Accounts)"); + Assert.True(result.IsSuccess); + + if (!isCachedData) + { + Assert.Empty(result.Errors); + } + else + { + Assert.Single(result.Errors); + var error = result.Errors.Single(); + Assert.Equal(ErrorSeverity.Warning, error.Severity); + } + + // Only shows warning if data source is passed directly to CountRows + result = engine.Check("CountRows(Filter(Accounts, IsBlank('Address 1: City')))"); + Assert.True(result.IsSuccess); + Assert.Empty(result.Errors); + } } } diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/Helpers/TestTabularDataSource.cs b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/Helpers/TestTabularDataSource.cs index 5d661e394a..cfc2945768 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/Helpers/TestTabularDataSource.cs +++ b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/Helpers/TestTabularDataSource.cs @@ -175,8 +175,9 @@ internal class TestDataSource : IExternalDataSource, IExternalTabularDataSource private readonly string[] _keyColumns; private readonly HashSet _selectableColumns; private readonly TabularDataQueryOptions _tabularDataQueryOptions; + private readonly bool _hasCachedCountRows; - internal TestDataSource(string name, DType schema, string[] keyColumns = null, IEnumerable selectableColumns = null) + internal TestDataSource(string name, DType schema, string[] keyColumns = null, IEnumerable selectableColumns = null, bool hasCachedCountRows = false) { ExternalDataEntityMetadataProvider = new ExternalDataEntityMetadataProvider(); Type = DType.AttachDataSourceInfo(schema, this); @@ -185,10 +186,13 @@ internal TestDataSource(string name, DType schema, string[] keyColumns = null, I _keyColumns = keyColumns ?? Array.Empty(); _selectableColumns = new HashSet(selectableColumns ?? Enumerable.Empty()); _tabularDataQueryOptions = new TabularDataQueryOptions(this); + _hasCachedCountRows = hasCachedCountRows; } public string Name { get; } + public bool HasCachedCountRows => this._hasCachedCountRows; + public virtual bool IsSelectable => true; public virtual bool IsDelegatable => throw new NotImplementedException();