fix compression utilities to avoid the file retrieval limit

src/python/dm/aps_beamline_tools/cli/compressFilesCli.py

@@ -22,6 +22,7 @@ class CompressFilesCli(ApsBeamlineCli):
     DEFAULT_COMPRESSION_METHOD = 'bz2'
     MAX_BATCH_SIZE = 100
     TASK_COMPLETION_WAIT_TIME = 10 # seconds
+    METADATA_BATCH_SIZE = 100000
 
 
     def __init__(self):
@@ -82,8 +83,19 @@ Description:
             queryDict['experimentFilePath'] = self.getExperimentFilePath()
         experimentName = self.getExperimentName()
         experiment = dsExperimentApi.getExperimentByName(experimentName)
-        fileMetadataList = catFileApi.getExperimentFiles(experimentName, queryDict)
-        fileMetadataList = [d for d in fileMetadataList if not d.get('compression')]
+        fileMetadataList = []
+        skip = 0
+        batch = 0
+        while True:
+            batch += 1
+            print('Retrieving experiment files, batch #%s' % batch)
+            fileMetadataList2 = catFileApi.getExperimentFiles(experimentName, queryDict, keyList=['experimentFilePath', 'id', 'compression'], skip=skip, limit=self.METADATA_BATCH_SIZE)
+            if len(fileMetadataList2) == 0:
+                break;
+            skip += self.METADATA_BATCH_SIZE
+            fileMetadataList2 = [d for d in fileMetadataList2 if not d.get('compression')]
+            if len(fileMetadataList2) > 0:
+                fileMetadataList = fileMetadataList + fileMetadataList2 
 
         # Check if there are files that satisfy the requirement.
         if not len(fileMetadataList):

src/python/dm/aps_beamline_tools/cli/decompressFilesCli.py

@@ -20,6 +20,7 @@ class DecompressFilesCli(ApsBeamlineCli):
 
     MAX_BATCH_SIZE = 100
     TASK_COMPLETION_WAIT_TIME = 10 # seconds
+    METADATA_BATCH_SIZE = 100000
 
     def __init__(self):
         ApsBeamlineCli.__init__(self, validArgCount=self.ANY_NUMBER_OF_POSITIONAL_ARGS)
@@ -72,8 +73,19 @@ Description:
             queryDict['experimentFilePath'] = self.getExperimentFilePath()
         experimentName = self.getExperimentName()
         experiment = dsExperimentApi.getExperimentByName(experimentName)
-        fileMetadataList = catFileApi.getExperimentFiles(experimentName, queryDict)
-        fileMetadataList = [d for d in fileMetadataList if d.get('compression')]
+        fileMetadataList = []
+        skip = 0
+        batch = 0
+        while True:
+            batch += 1
+            print('Retrieving experiment files, batch #%s' % batch)
+            fileMetadataList2 = catFileApi.getExperimentFiles(experimentName, queryDict, keyList=['experimentFilePath', 'id', 'compression'], skip=skip, limit=self.METADATA_BATCH_SIZE)
+            if len(fileMetadataList2) == 0:
+                break;
+            skip += self.METADATA_BATCH_SIZE
+            fileMetadataList2 = [d for d in fileMetadataList2 if d.get('compression')]
+            if len(fileMetadataList2) > 0:
+                fileMetadataList = fileMetadataList + fileMetadataList2 
 
         # Check if there are files that satisfy the requirement.
         if not len(fileMetadataList):