Issue #3753 sls min object size to transit (#3754)

epam · Oct 24, 2024 · 4c537d3 · 4c537d3
1 parent 8d004c2
commit 4c537d3
Show file tree

Hide file tree

Showing 10 changed files with 147 additions and 301 deletions.
diff --git a/storage-lifecycle-service/integrational_tests/decorator/cloud_decator.py b/storage-lifecycle-service/integrational_tests/decorator/cloud_decator.py
@@ -28,8 +28,8 @@ def __init__(self, cloud_operations, testcase):
             for file in storage.files:
                 self.watched_files_by_storages[storage.storage][file.path if file.path.startswith("/") else "/" + file.path] = file
 
-    def prepare_bucket_if_needed(self, region, storage_container):
-        self.cloud_operations.prepare_bucket_if_needed(region, storage_container)
+    def prepare_bucket_if_needed(self, region, storage_container, object_size_for_transit=None):
+        self.cloud_operations.prepare_bucket_if_needed(region, storage_container, object_size_for_transit)
 
     def list_objects_by_prefix(self, region, storage_container, classes_to_list=None,
                                list_versions=False, convert_paths=True):

diff --git a/...rces/testcases/case_38_nothing_eligable_to_do_because_files_are_small_for_glacier_ir.json b/...rces/testcases/case_38_nothing_eligable_to_do_because_files_are_small_for_glacier_ir.json
diff --git a/...testcases/case_39_action_should_succeed_because_other_files_are_small_for_glacier_ir.json b/...testcases/case_39_action_should_succeed_because_other_files_are_small_for_glacier_ir.json
diff --git a/..._40_action_to_glacer_ir_should_start_only_for_one_file_because_other_files_are_small.json b/..._40_action_to_glacer_ir_should_start_only_for_one_file_because_other_files_are_small.json
diff --git a/storage-lifecycle-service/sls/app/cloud_storage_adapter.py b/storage-lifecycle-service/sls/app/cloud_storage_adapter.py
@@ -20,6 +20,7 @@
 # If the metadata with such key is present on the storage,
 # StorageLifecycleArchivingSynchronizer will skip such storage
 SKIP_ARCHIVING_TAG = "storage_skip_archiving_tag"
+MINIMUM_OBJECT_SIZE_FOR_TRANSIT = "minimum_object_size_for_transit"
 
 
 def _verify_s3_sls_properties(sls_properties, logger):
@@ -114,11 +115,25 @@ def should_be_skipped(self, storage):
             return False
         return tag_to_skip_storage in metadata
 
+    def get_minimum_object_size_for_transit(self, storage):
+        region = self.fetch_region(storage.region_id)
+        return self._get_minimum_object_size_for_transit(region)
+
+    def _get_minimum_object_size_for_transit(self, region):
+        value = region.storage_lifecycle_service_properties.get(MINIMUM_OBJECT_SIZE_FOR_TRANSIT) if region else None
+        if isinstance(value, str) and value.isnumeric() or isinstance(value, int):
+            return int(value)
+        else:
+            self.logger.log("Bad value for {}: {}, should be non-nagative int. "
+                            "Returning None".format(MINIMUM_OBJECT_SIZE_FOR_TRANSIT, value))
+            return None
+
     def prepare_bucket_if_needed(self, storage):
         region = self.fetch_region(storage.region_id)
         storage_cloud_identifier, storage_path_prefix = self._parse_storage_path(storage)
         storage_container = CloudPipelineStorageContainer(storage_cloud_identifier, storage_path_prefix, storage)
-        self.cloud_operations[storage.storage_type].prepare_bucket_if_needed(region, storage_container)
+        object_size_for_transit = self._get_minimum_object_size_for_transit(region)
+        self.cloud_operations[storage.storage_type].prepare_bucket_if_needed(region, storage_container, object_size_for_transit)
 
     def list_objects_by_prefix(self, storage, prefix, classes_to_list):
         region = self.fetch_region(storage.region_id)