Merge pull request #67 from Dysio/APM-314207-dtcli-improvements

Download schemas and wipe extension options added

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.0.0
+current_version = 1.1.0-alpha.1
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-(?P<release>.*))?
 serialize = 
 	{major}.{minor}.{patch}-{release}

.gitignore

@@ -9,6 +9,12 @@ extension/
 *.key
 *.crt
 
+# Downloaded schemas
+schemas/
+
+# Secrets
+secrets/
+
 # Git
 .gitconfig
 

README.md

@@ -40,6 +40,22 @@ See: [Add your root certificate to the Dynatrace credential vault](https://www.d
 ```
 Use `dt extension --help` to learn more
 
+4. Download extension schemas
+```sh
+  dt extension schemas
+```
+_API permissions needed: `extensions.read`_
+
+This script should only be needed once, whenever schema files are missing or you want to target a different version than what you already have. It does the following:
+* Downloads all the extension schema files of a specific version
+* Schemas are downloaded to `schemas` folder
+
+5. Wipes out extension from Dynatrace Cluster
+```sh
+  dt extension delete
+```
+Use `dt extension --help` to learn more
+
 
 ## Using dt-cli from your Python code
 

dtcli/__init__.py

@@ -13,3 +13,4 @@
 # limitations under the License.
 
 from .version import __version__
+from dtcli import scripts

dtcli/__main__.py

@@ -0,0 +1,4 @@
+import dtcli
+
+if __name__ == "__main__":
+    dtcli.scripts.dt.main()

dtcli/api.py

@@ -0,0 +1,134 @@
+import json
+import os
+import requests as _requests_impl
+import zipfile, io
+
+# TODO: support pagination
+
+class DynatraceAPIClient:
+    def __init__(self, url, token, requests = None):
+        self.url_base = url
+        self.headers = {"Authorization": f"Api-Token {token}"}
+        self.requests = requests if requests is not None else _requests_impl
+
+    def acquire_monitoring_configurations(self, fqdn: str):
+        r = self.requests.get(self.url_base + f"/api/v2/extensions/{fqdn}/monitoringConfigurations", headers=self.headers)
+        r.raise_for_status()
+        return r.json()["items"]
+
+    def acquire_environment_configuration(self, fqdn: str):
+        r = self.requests.get(self.url_base + f"/api/v2/extensions/{fqdn}/environmentConfiguration", headers=self.headers)
+
+        if r.status_code == 404:
+            return
+
+        r.raise_for_status()
+        return r.json()
+
+    def acquire_extensions(self):
+        r = self.requests.get(self.url_base + f"/api/v2/extensions", headers=self.headers)
+        r.raise_for_status()
+        return r.json()["extensions"]
+
+    def acquire_extension_versions(self, fqdn: str):
+        r = self.requests.get(self.url_base + f"/api/v2/extensions/{fqdn}", headers=self.headers)
+
+        r.raise_for_status()
+        return r.json()["extensions"]
+
+    def delete_monitoring_configuration(self, fqdn: str, configuration_id: str):
+        r = self.requests.delete(self.url_base + f"/api/v2/extensions/{fqdn}/monitoringConfigurations/{configuration_id}", headers=self.headers)
+        try:
+            r.raise_for_status()
+        except:
+            err = ""
+            try:
+                err = r.json()
+            except:
+                pass
+
+            print(err)
+            raise
+
+    def delete_environment_configuration(self, fqdn: str):
+        r = self.requests.delete(self.url_base + f"/api/v2/extensions/{fqdn}/environmentConfiguration", headers=self.headers)
+        err = r.json()
+        try:
+            r.raise_for_status()
+        except:
+            print(err)
+            if r.code != 404:
+                raise
+
+    def delete_extension(self, fqdn: str, version: str):
+        r = self.requests.delete(self.url_base + f"/api/v2/extensions/{fqdn}/{version}", headers=self.headers)
+        err = r.json()
+        try:
+            r.raise_for_status()
+        except:
+            print(err)
+            if r.code != 404:
+                raise
+
+    def get_schema_target_version(self, target_version: str):
+        """Get version number from tenant. If version doesn't exist return list of available versions."""
+        r = self.requests.get(self.url_base + "/api/v2/extensions/schemas", headers=self.headers)
+        r.raise_for_status()
+        versions = r.json().get("versions", [])
+
+        if target_version == "latest":
+            return versions[-1]
+
+        matches = [v for v in versions if v.startswith(target_version)]
+        if matches:
+            return matches[0]
+
+        raise SystemExit(f"Target version {target_version} does not exist. \nAvailable versions: {versions}")
+
+    def download_schemas(self, target_version: str, download_dir: str):
+        """Downloads schemas from choosen version"""
+
+        version = self.get_schema_target_version(target_version)
+
+        if not os.path.exists(download_dir):
+            os.makedirs(download_dir)
+
+        header = self.headers
+        header["accept"] = "application/octet-stream"
+        file = self.requests.get(self.url_base + f"/api/v2/extensions/schemas/{version}", headers=header, stream=True)
+        zfile = zipfile.ZipFile(io.BytesIO(file.content))
+
+        THRESHOLD_ENTRIES = 10000
+        THRESHOLD_SIZE = 1000000000
+        THRESHOLD_RATIO = 10
+
+        totalSizeArchive = 0
+        totalEntryArchive = 0
+
+        for zinfo in zfile.infolist():
+            data = zfile.read(zinfo)
+            totalEntryArchive += 1
+            totalSizeArchive = totalSizeArchive + len(data)
+            ratio = len(data) / zinfo.compress_size
+            if ratio > THRESHOLD_RATIO:
+                raise Exception("ratio between compressed and uncompressed data is highly suspicious, looks like a Zip Bomb Attack")
+
+            if totalSizeArchive > THRESHOLD_SIZE:
+                raise Exception("the uncompressed data size is too much for the application resource capacity")
+
+            if totalEntryArchive > THRESHOLD_ENTRIES:
+                raise Exception("too much entries in this archive, can lead to inodes exhaustion of the system")
+
+        zfile.extractall(download_dir)
+        zfile.close()
+
+        return version
+
+    def point_environment_configuration_to(self, fqdn: str, version: str):
+        r = self.requests.put(self.url_base + f"/api/v2/extensions/{fqdn}/environmentConfiguration", headers=self.headers, json={"version": version})
+        err = r.json()
+        try:
+            r.raise_for_status()
+        except:
+            print(err)
+            raise

dtcli/constants.py

@@ -25,3 +25,5 @@
 DEFAULT_CA_CERT = os.path.join(os.path.curdir, "ca.pem")
 DEFAULT_CA_KEY = os.path.join(os.path.curdir, "ca.key")
 DEFAULT_CERT_VALIDITY = 365 * 3
+DEFAULT_SCHEMAS_DOWNLOAD_DIR = os.path.join(os.path.curdir, "schemas")
+DEFAULT_TOKEN_PATH = "./secrets/token"

dtcli/delete_extension.py

@@ -0,0 +1,136 @@
+import json
+from collections import defaultdict
+from typing import Dict, Set, Optional, List
+from pathlib import Path
+
+
+from dtcli.api import DynatraceAPIClient
+
+class State:
+    def __init__(self, d):
+        self.d = d
+
+    def __getitem__(self,key):
+        return self.d[key]
+
+    def __contains__(self, key):
+        return key in self.d
+
+    def __str__(self):
+        return str(self.d)
+
+    def versions(self, extension_fqdn, exclude: Optional[Set[str]] = None) -> List[str]:
+        if exclude is None:
+            exclude = set()
+
+        all_versions: Set[str] = set(self[extension_fqdn].keys())
+
+        return list(sorted(all_versions - exclude))
+
+    def as_dict(self):
+        return self.d
+
+def acquire_state(client: DynatraceAPIClient) -> State:
+    extensions_listing = list(map(lambda e: e["extensionName"], client.acquire_extensions()))
+
+    extensions_data = []
+    for e in extensions_listing:
+        _extensions_data = client.acquire_extension_versions(e)
+        extensions_data += _extensions_data
+
+    extensions = defaultdict(dict)
+    for e in extensions_data:
+        name, version = e["extensionName"], e["version"]
+        extensions[name][version] = {"monitoring_configurations": []}
+
+    for extension in extensions_listing:
+        environment_configuration = client.acquire_environment_configuration(extension)
+        monitoring_configurations = client.acquire_monitoring_configurations(extension)
+
+        if environment_configuration:
+            extensions[extension][environment_configuration["version"]]["environment_configuration"] = environment_configuration
+            for mc in monitoring_configurations:
+                extensions[extension][mc["value"]["version"]]["monitoring_configurations"].append(mc)
+
+    s = State(extensions)
+    return s
+
+def acquire_state_for_extension(client: DynatraceAPIClient, extension: str) -> State:
+    extensions_listing = list(map(lambda e: e["extensionName"], client.acquire_extensions()))
+
+    if extension not in extensions_listing:
+        raise Exception("Extension doesn't exist")
+
+    versions = client.acquire_extension_versions(extension)
+    extension_data = defaultdict(dict)
+    for e in versions:
+        name, version = e["extensionName"], e["version"]
+        extension_data[name][version] = {"monitoring_configurations": []}
+
+    environment_configuration = client.acquire_environment_configuration(extension)
+    monitoring_configurations = client.acquire_monitoring_configurations(extension)
+
+    if environment_configuration:
+        extension_data[extension][environment_configuration["version"]]["environment_configuration"] = environment_configuration
+        for mc in monitoring_configurations:
+            extension_data[extension][mc["value"]["version"]]["monitoring_configurations"].append(mc)
+
+    state = State(extension_data)
+    return state
+
+def wipe_extension_version(client, state, extension_fqdn: str, version: str):
+    assert extension_fqdn in state
+    if version not in state[extension_fqdn]:
+        return
+
+    # TODO: when refactoring to command pattern remember that the order and groups matter
+    for mc in state[extension_fqdn][version]["monitoring_configurations"]:
+        client.delete_monitoring_configuration(extension_fqdn, mc["objectId"])
+    if "environment_configuration" in state[extension_fqdn][version]:
+        # TODO: dehardcode it
+        there_are_other_mcs = False
+
+        if there_are_other_mcs:
+            # this will be a pain to sensibly parallelize, so... for now don't run this thing on the same fqdn simultaneously
+            target_version = state.versions(extension_fqdn, exclude={version})[-1]
+            client.point_environment_configuration_to(extension_fqdn, target_version)
+        else:
+            client.delete_environment_configuration(extension_fqdn)
+
+    client.delete_extension(extension_fqdn, version)
+
+def wipe_extension(client, state, extension_fqdn: str):
+    if extension_fqdn not in state:
+        return
+
+    env_conf_ver = client.acquire_environment_configuration(extension_fqdn)["version"]
+    versions = [v for v in state[extension_fqdn]]
+
+    wipe_extension_version(client, state, extension_fqdn, env_conf_ver)
+    versions.remove(env_conf_ver)
+
+    for version in versions:
+        wipe_extension_version(client, state, extension_fqdn, version)
+
+
+# TODO: split arguments that will be usefull with all commands (tenant, secrets)
+def wipe_single_version(fqdn: str, version: str, tenant: str, token_path: str):
+    """Wipe single extension version
+
+    Example: ... 'com.dynatrace.palo-alto.generic' '0.1.5' --tenant lwp00649 --secrets-path ./secrets
+    """
+    with open(token_path) as f:
+        token = f.readlines()[0].rstrip()
+
+    client = DynatraceAPIClient(tenant, token)
+    state = acquire_state(client)
+    print(state)
+
+    wipe_extension_version(client, state, fqdn, version)
+
+def wipe(fqdn: str, tenant: str, token: str):
+    # TODO: move client creation further up the chain
+    client = DynatraceAPIClient(tenant, token)
+    state = acquire_state_for_extension(client, fqdn)
+
+    wipe_extension(client, state, fqdn)

dtcli/scripts/__init__.py

@@ -10,4 +10,7 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.
+
+from dtcli.scripts import dt
+