feat: allow wildcards in .uccignore file (#1012)

Currently it is not possible to put wildcards in the .uccignore file,
this can be problematic if you do not know the exact filename that needs
to be ignored, for example a developer may be developing a TA on a Mac
which could create darwin related binaries in the lib/<packageName>
folder.
Other "ignore" files, such as .gitignore and .dockerignore allow for
wildcards and therefore developers may expect that same with uccignore.

.uccignore:
```
...
lib/**/*darwin*.so
```

Before:
```
...
WARNING: While ignoring the files mentioned in .uccignore '<projectDir>/output/<appName>/lib/**/*darwin*.so was not found
INFO: Removed ['<otherRemovedFile>', '<projectDir>/output/<appName>/lib/**/*darwin*.so'] files
...
```
Note, in the above it does not expand the wildcards, the WARNING says
that there was no match, however the INFO contradicts and suggests that
it has been removed (which it has not).

After:
```
...
INFO: Removed ['<projectDir>/output/<appName>/lib/_cffi_backend.cpython-39-darwin.so', '<projectDir>/output/<appName>/lib/charset_normalizer/md__mypyc.cpython-39-darwin.so', '<projectDir>/output/<appName>/lib/charset_normalizer/md.cpython-39-darwin.so'] files
...
```
Note: The list of removed files is now representative of what was
removed, rather than the wildcard, so the developer can see exactly what
has been ignored/removed.

This also fixes #1011

---------

Co-authored-by: sgoral <sgoral@splunk.com>

docs/uccignore.md

@@ -5,4 +5,28 @@ add-on recursively overrides the output folder.
 
 It is expected to be placed in the same folder as `globalConfig` file to have effect.
 
-You will see a warning message in case ignored file is not found in the output folder.
+Uccignore supports wildcard expressions, thanks to which we can find all files matching a specific pattern.
+
+e.g. for given file structure 
+
+```
+...
+└── lib
+    └── 3rdparty
+    │   ├── linux
+    │   ├   └── pycache.pyc
+    │   ├── linux_with_deps
+    │   ├   └── pycache.pyc
+    │   └── windows
+    │       └── pycache.pyc
+    └── requests
+    │       └── pycache.pyc
+    └── urllib
+            └── pycache.pyc
+```  
+
+we can remove all `.pyc` files by adding `lib/**/pycache.pyc` to the .uccignore file. 
+If we want to remove all `.pyc` files just from the `3rdparty` directory, we need to change pattern to `lib/3rdparty/**/pycache.pyc`.
+If we want to remove only for one specific platform, we need to provide the exact path e.g. **`lib/3rdparty/windows/pycache.pyc`**.
+
+In case no file is found for the specified pattern, you will see an appropriate warning message.

splunk_add_on_ucc_framework/commands/build.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 import configparser
+import glob
 import json
 import logging
 import os
@@ -195,30 +196,32 @@ def _get_ignore_list(
                 os.path.join(output_directory, addon_name, utils.get_os_path(path))
             ).strip()
             for path in ignore_list
+            if path.strip()
         ]
         return ignore_list
 
 
-def _remove_listed_files(ignore_list: List[str]) -> None:
+def _remove_listed_files(ignore_list: List[str]) -> List[str]:
     """
     Return path of files/folders to removed in output folder.
 
     Args:
-        ignore_list (list): List of files/folder to removed in output directory.
-
+        ignore_list (list): List of files/folder patterns to be removed in output directory.
     """
-    for path in ignore_list:
-        if os.path.exists(path):
-            if os.path.isfile(path):
-                os.remove(path)
-            elif os.path.isdir(path):
-                shutil.rmtree(path, ignore_errors=True)
-        else:
-            logger.warning(
-                "While ignoring the files mentioned in .uccignore {} was not found".format(
-                    path
-                )
-            )
+    removed_list = []
+    for pattern in ignore_list:
+        paths = glob.glob(pattern, recursive=True)
+        if not paths:
+            logger.warning(f"No files found for the specified pattern: {pattern}")
+            continue
+        for path in paths:
+            if os.path.exists(path):
+                if os.path.isfile(path):
+                    os.remove(path)
+                elif os.path.isdir(path):
+                    shutil.rmtree(path, ignore_errors=True)
+                removed_list.append(path)
+    return removed_list
 
 
 def generate_data_ui(
@@ -629,9 +632,9 @@ def generate(
         os.path.abspath(os.path.join(source, os.pardir, ".uccignore")),
         output_directory,
     )
-    _remove_listed_files(ignore_list)
-    if ignore_list:
-        logger.info(f"Removed {ignore_list} files")
+    removed_list = _remove_listed_files(ignore_list)
+    if removed_list:
+        logger.info("Removed:\n{}".format("\n".join(removed_list)))
     utils.recursive_overwrite(source, os.path.join(output_directory, ta_name))
     logger.info("Copied package directory")
 

tests/smoke/test_ucc_build.py

@@ -429,3 +429,91 @@ def summarize_types(raw_expected_logs):
         # summary messages must be the same but might come in different order
         assert log_line.message in expected_logs.keys()
         assert log_line.levelname == expected_logs[log_line.message]
+
+
+@pytest.mark.skipif(sys.version_info >= (3, 8), reason=PYTEST_SKIP_REASON)
+def test_ucc_generate_with_everything_uccignore(caplog):
+    with tempfile.TemporaryDirectory() as temp_dir:
+        package_folder = path.join(
+            path.dirname(path.realpath(__file__)),
+            "..",
+            "testdata",
+            "test_addons",
+            "package_global_config_everything_uccignore",
+            "package",
+        )
+        build.generate(source=package_folder, output_directory=temp_dir)
+
+        expected_warning_msg = (
+            f"No files found for the specified pattern: "
+            f"{temp_dir}/Splunk_TA_UCCExample/bin/wrong_pattern"
+        )
+
+        edm1 = "Removed:"
+        edm2 = f"\n{temp_dir}/Splunk_TA_UCCExample/bin/splunk_ta_uccexample_rh_example_input_one.py"
+        edm3 = f"\n{temp_dir}/Splunk_TA_UCCExample/bin/example_input_one.py"
+        edm4 = f"\n{temp_dir}/Splunk_TA_UCCExample/bin/splunk_ta_uccexample_rh_example_input_two.py"
+
+        assert expected_warning_msg in caplog.text
+        assert (edm1 + edm2 + edm3 + edm4) in caplog.text or (
+            edm1 + edm3 + edm2 + edm4
+        ) in caplog.text
+
+        expected_folder = path.join(
+            path.dirname(__file__),
+            "..",
+            "testdata",
+            "expected_addons",
+            "expected_output_global_config_everything",
+            "Splunk_TA_UCCExample",
+        )
+        actual_folder = path.join(temp_dir, "Splunk_TA_UCCExample")
+        _compare_app_conf(expected_folder, actual_folder)
+        files_to_be_equal = [
+            ("README.txt",),
+            ("default", "alert_actions.conf"),
+            ("default", "eventtypes.conf"),
+            ("default", "inputs.conf"),
+            ("default", "restmap.conf"),
+            ("default", "tags.conf"),
+            ("default", "splunk_ta_uccexample_settings.conf"),
+            ("default", "web.conf"),
+            ("default", "server.conf"),
+            ("default", "data", "ui", "alerts", "test_alert.html"),
+            ("default", "data", "ui", "nav", "default.xml"),
+            ("default", "data", "ui", "views", "configuration.xml"),
+            ("default", "data", "ui", "views", "inputs.xml"),
+            ("default", "data", "ui", "views", "dashboard.xml"),
+            ("default", "data", "ui", "views", "splunk_ta_uccexample_redirect.xml"),
+            ("bin", "splunk_ta_uccexample", "modalert_test_alert_helper.py"),
+            ("bin", "example_input_two.py"),
+            ("bin", "example_input_three.py"),
+            ("bin", "example_input_four.py"),
+            ("bin", "import_declare_test.py"),
+            ("bin", "splunk_ta_uccexample_rh_account.py"),
+            ("bin", "splunk_ta_uccexample_rh_three_custom.py"),
+            ("bin", "splunk_ta_uccexample_rh_example_input_four.py"),
+            ("bin", "splunk_ta_uccexample_custom_rh.py"),
+            ("bin", "splunk_ta_uccexample_rh_oauth.py"),
+            ("bin", "splunk_ta_uccexample_rh_settings.py"),
+            ("bin", "test_alert.py"),
+            ("README", "alert_actions.conf.spec"),
+            ("README", "inputs.conf.spec"),
+            ("README", "splunk_ta_uccexample_account.conf.spec"),
+            ("README", "splunk_ta_uccexample_settings.conf.spec"),
+            ("metadata", "default.meta"),
+        ]
+        helpers.compare_file_content(
+            files_to_be_equal,
+            expected_folder,
+            actual_folder,
+        )
+        files_to_exist = [
+            ("static", "appIcon.png"),
+            ("static", "appIcon_2x.png"),
+            ("static", "appIconAlt.png"),
+            ("static", "appIconAlt_2x.png"),
+        ]
+        for f in files_to_exist:
+            expected_file_path = path.join(expected_folder, *f)
+            assert path.exists(expected_file_path)

tests/testdata/test_addons/package_global_config_everything_uccignore/.uccignore

@@ -0,0 +1,4 @@
+**/**one.py
+
+bin/splunk_ta_uccexample_rh_example_input_two.py
+bin/wrong_pattern

tests/testdata/test_addons/package_global_config_everything_uccignore/LICENSES/Apache-2.0.txt

@@ -0,0 +1 @@
+dummy apache license