add last_modified to file listing, support for dryrun, expand user pa…

…ths and return list of deleted files

pyepic/client/base.py

@@ -52,6 +52,9 @@ def __init__(
             host=connection_url,
             api_key={"Bearer": "Bearer {}".format(connection_token)},
         )
+
+    def set_limt(self, limit):
+        self.LIMIT = limit
 
 
 class EPICClient(object):

pyepic/client/catalog.py

@@ -83,6 +83,21 @@ def list_clusters(self, cluster_name=None, queue_name=None, application_id=None)
                 for result in results.results:
                     yield result
 
+    def queue_details(self, queue_id):
+        """Get the details of queue with id queue_id
+
+        :param queue_id: ID of queue to get details for
+        :type queue_id: int
+
+        :return: BatchQueueDetails
+        :rtype: :class:`epiccore.models.BatchQueueDetails`
+        """
+        with epiccore.ApiClient(self.configuration) as api_client:
+            instance = epiccore.CatalogApi(api_client)
+            result = instance.catalog_clusters_read(queue_id
+            )
+            return result
+
     def list_applications(self, product_name=None):
         """List the applications available in EPIC
 
@@ -109,6 +124,20 @@ def list_applications(self, product_name=None):
                 for result in results.results:
                     yield result
 
+    def application_details(self, application_id):
+        """Get the details of application with id application_id
+
+        :param application_id: ID of application to get details for
+        :type application_id: int
+
+        :return: BatchQueueDetails
+        :rtype: :class:`epiccore.models.BatchApplicationDetails`
+        """
+        with epiccore.ApiClient(self.configuration) as api_client:
+            instance = epiccore.CatalogApi(api_client)
+            result = instance.catalog_applications_read(application_id)
+            return result
+
     def list_desktops(self):
         """List the available Desktops in EPIC
 

pyepic/client/data.py

@@ -198,14 +198,17 @@ class DataObject(object):
     :type folder: bool
     :param name: Size of the object if available
     :type name: int
+    :param last_modified: Last modified time if available. Datetime in ISO 8601 format, UTC timezone.
+    :type last_modified: str
     """
 
-    def __init__(self, name, obj_path, folder=False, size=None):
+    def __init__(self, name, obj_path, folder=False, size=None, last_modified=None):
         """Constructor method"""
         self.name = name
         self.obj_path = obj_path
         self.folder = folder
         self.size = size
+        self.last_modified = last_modified
 
 
 class DataClient(Client):
@@ -319,6 +322,7 @@ def ls(self, epic_path):
                     self._s3_to_epic_path(item["Key"]),
                     folder=False,
                     size=item["Size"],
+                    last_modified=item['LastModified'].isoformat()
                 )
                 yield file
 
@@ -360,35 +364,41 @@ def upload_file(self, file, epic_path):
             s3_path = self._epic_path_to_s3(epic_path)
             self._s3_client.upload_fileobj(file, self._s3_bucket, s3_path)
 
-    def delete(self, epic_path):
+    def delete(self, epic_path, dryrun=False):
         """
         Delete the file of folder at epic_path
             :param epic_path: Path of a file or folder to delete in the form epic://[<folder>]/<file>
             :type epic_path: str
+            :param dryrun: If dryrun is True then return a list of files that would be deleted without actually deleting them
+            :type dryrun: bool
 
-            :return: Was the delete successful, for folder deletions a False returns means one or more delete failures
-            :rtype: bool
+            :return: List of the files deleted
+            :rtype: List[str]
         """
         self._connect()
+        deleted = []
         if not epic_path.endswith("/"):
             key = self._epic_path_to_s3(epic_path)
-            response = self._s3_client.delete_objects(Bucket=self._s3_bucket, Key=key)
-            if response["DeleteMarker"]:
-                return True
-            return False
+            deleted.append(epic_path)
+            if not dryrun:
+                response = self._s3_client.delete_object(Bucket=self._s3_bucket, Key=key)
+            return deleted
         else:
             objects = []
             prefix = self._epic_path_to_s3(epic_path)
             key_list = self._list_contents(prefix)
             for item in key_list:
+                deleted.append(self._s3_to_epic_path(item))
                 objects.append({"Key": item})
-            response = self._s3_client.delete_objects(
-                Bucket=self._s3_bucket,
-                Delete={"Objects": objects},
-            )
-            if "Errors" in response:
-                return False
-            return True
+            if not dryrun:
+                response = self._s3_client.delete_objects(
+                    Bucket=self._s3_bucket,
+                    Delete={"Objects": objects},
+                )
+                if "Errors" in response:
+                    # TODO, remove any error files for deleted list
+                    pass
+            return deleted
 
     def sync(
         self,
@@ -422,6 +432,7 @@ def sync(
                 raise ValueError("Both source_path and target_path are EPIC paths")
             if not source_path.endswith("/"):
                 source_path = source_path + "/"
+            target_path = os.path.expanduser(target_path)
             Path(target_path).mkdir(parents=True, exist_ok=True)
             prefix = self._epic_path_to_s3(source_path)
             self._download(
@@ -437,6 +448,7 @@ def sync(
                 raise ValueError("Both source_path and target_path are EPIC paths")
             if not target_path.endswith("/"):
                 target_path = target_path + "/"
+            source_path = os.path.expanduser(source_path)
             if not os.path.isdir(source_path):
                 raise ValueError("source_path does not exist")
             prefix = self._epic_path_to_s3(target_path)

pyepic/client/job.py

@@ -69,14 +69,15 @@ def submit(self, job_array_spec):
             instance = epiccore.JobApi(api_client)
             return instance.job_create(job_array_spec)
 
-    def list(self):
+    def list(self, limit=10):
         """List all of the jobs in EPIC.
 
+        :param limit: Maximum number of jobs to list
+        :type int
         :return: Iterable collection of Jobs
         :rtype: collections.Iterable[:class:`epiccore.models.Job`]
         """
         with epiccore.ApiClient(self.configuration) as api_client:
-            limit = self.LIMIT
             offset = 0
             instance = epiccore.JobApi(api_client)
             results = instance.job_list(limit=limit, offset=offset)