Use S3 native CRC64NVME full-object checksums (v8.0.0)

README.md

@@ -48,7 +48,7 @@ Full documentation available on [Read the Docs](https://awsimple.readthedocs.io/
 - Convert back and forth between DynamoDB items and Python dictionaries automatically. Converts many common data types to DynamoDB compatible types,
 including nested structures, sets, images (PIL), and Enum/StrEnum.
 
-- True file hashing (SHA512) for S3 files (S3's etag is not a true file hash).
+- Full-object checksums (CRC64NVME) for S3 files (S3's etag is not a true file hash).
 
 - Supports moto mock and localstack. Handy for testing and CI.
 
@@ -106,8 +106,14 @@ level API to AWS services (such as S3, DynamoDB, SNS, and SQS) to improve progra
 
 ## S3
 
-`awsimple` calculates the local file hash (sha512) and inserts it into the S3 object metadata.  This is used
-to test for file equivalency.
+`awsimple` uses S3's native full-object checksums to test for file equivalency (e.g. to avoid re-uploading
+unchanged files). Uploads use CRC64NVME - the algorithm S3 itself defaults to, and the only one S3 computes
+as a full-object value for both single-part and multipart uploads - and S3 validates the data against the
+checksum server-side before storing. Objects written by other tools with a native full-object SHA-512
+checksum are also recognized and compared. Objects written by older awsimple versions (which stored a
+SHA-512 in custom object metadata) are detected and always re-uploaded, so every object awsimple writes
+ends up with the native checksum. Objects without any recognizable full-object hash fall back to
+modification-time and file-size comparison.
 
 ## Caching
 

awsimple/__version__.py

@@ -1,7 +1,7 @@
 __application_name__ = "awsimple"
 __title__ = __application_name__
 __author__ = "abel"
-__version__ = "7.3.0"
+__version__ = "8.0.0"
 __author_email__ = "j@abel.co"
 __url__ = "https://github.com/jamesabel/awsimple"
 __download_url__ = "https://github.com/jamesabel/awsimple"

awsimple/s3.py

@@ -2,14 +2,15 @@
 S3 Access
 """
 
+import base64
 import os
 import shutil
 import time
 from math import isclose
 from pathlib import Path
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Dict, List, Union
+from typing import Any, Dict, List, Union
 import json
 from logging import getLogger
 
@@ -19,7 +20,7 @@
 from s3transfer import S3UploadFailedError
 import urllib3.exceptions
 from typeguard import typechecked
-from hashy import get_string_sha512, get_file_sha512, get_bytes_sha512, get_dls_sha512
+from hashy import get_file_sha512, get_bytes_sha512, get_dls_sha512, get_file_crc64nvme, get_bytes_crc64nvme
 from yasf import sf
 
 from awsimple import (
@@ -34,8 +35,16 @@
 )
 
 # Use this project's name as a prefix to avoid string collisions.  Use dashes instead of underscore since that's AWS's convention.
+# This custom metadata predates S3's native SHA-512 checksum support (April 2026). It is no longer written - it is only read to
+# detect objects written by older awsimple versions, which are always re-uploaded so that every object gains the native checksum.
 sha512_string = f"{__application_name__}-sha512"
 
+# Uploads at or above this size use multipart (the boto3 default threshold). awsimple uses CRC64NVME checksums everywhere since
+# it's the only checksum family S3 computes as a full-object value for both single-part and multipart uploads (SHA-family multipart
+# checksums are composite and can't be compared to a local file hash), and it's also S3's own default - so objects written by other
+# modern tools are usually comparable too.
+default_multipart_threshold = 8 * 1024 * 1024  # 8 MiB
+
 json_extension = ".json"
 
 log = getLogger(__application_name__)
@@ -67,22 +76,26 @@ class S3ObjectMetadata:
     size: int
     mtime: datetime
     etag: str  # generally not used
-    sha512: Union[str, None]  # hex string - only entries written with awsimple will have this
+    sha512: Union[str, None]  # hex string - S3's native full-object SHA-512 checksum (None if the object doesn't have one, e.g. multipart uploads or objects written by other tools)
     url: str  # URL of S3 object
+    legacy_sha512: Union[str, None] = None  # hex string from awsimple's legacy custom metadata - only present on objects written by awsimple versions before native checksum support
+    crc64nvme: Union[str, None] = None  # hex string - S3's native full-object CRC64NVME checksum (awsimple uses this for multipart uploads)
 
     def get_sha512(self) -> str:
         """
-        Get hash used to compare S3 objects. If the SHA512 is available (recommended), then use that. If not (e.g. an S3 object wasn't written with AWSimple), create a "substitute"
-        SHA512 hash that should change if the object contents change.
-        :return: SHA512 hash (as string)
+        Get a content-derived value used to compare and cache S3 objects. If the native SHA512 is available (recommended), then use that. Otherwise use the native full-object
+        CRC64NVME (e.g. multipart uploads). If neither is available (e.g. an S3 object wasn't written with AWSimple), create a "substitute" hash that should change if the object
+        contents change.
+        :return: hash (as string)
         """
         if (sha512_value := self.sha512) is None:
-            # round timestamp to seconds to try to avoid possible small deltas when dealing with time and floats
-            mtime_as_int = int(round(self.mtime.timestamp()))
-            metadata_list = [self.bucket, self.key, self.size, mtime_as_int]
-            if self.etag is not None and len(self.etag) > 0:
-                metadata_list.append(self.etag)
-            sha512_value = get_dls_sha512(metadata_list)
+            if (sha512_value := self.crc64nvme) is None:
+                # round timestamp to seconds to try to avoid possible small deltas when dealing with time and floats
+                mtime_as_int = int(round(self.mtime.timestamp()))
+                metadata_list = [self.bucket, self.key, self.size, mtime_as_int]
+                if self.etag is not None and len(self.etag) > 0:
+                    metadata_list.append(self.etag)
+                sha512_value = get_dls_sha512(metadata_list)
 
         return sha512_value
 
@@ -92,6 +105,20 @@ def serializable_object_to_json_as_bytes(json_serializable_object: Union[List, D
     return bytes(json.dumps(json_serializable_object, default=convert_serializable_special_cases).encode("UTF-8"))
 
 
+@typechecked()
+def _native_checksum_to_hex(checksum_base64: Union[str, None]) -> Union[str, None]:
+    """
+    Convert an S3 native checksum (base64 encoded) to a hex string comparable with locally computed hashes.
+    Composite (multipart) checksums have a "-<part count>" suffix and do not represent the whole object, so they convert to None.
+
+    :param checksum_base64: base64 encoded checksum from S3 (or None)
+    :return: hex string, or None if no full-object checksum available
+    """
+    if checksum_base64 is None or "-" in checksum_base64:
+        return None
+    return base64.b64decode(checksum_base64).hex()
+
+
 def _get_json_key(s3_key: str):
     """
     get JSON key given an s3_key that may not have the .json extension
@@ -120,10 +147,23 @@ def __init__(self, bucket_name: Union[str, None] = None, **kwargs):
         self._bucket_region = None  # type: Union[str, None]  # lazily determined and cached
         super().__init__(resource_name="s3", **kwargs)
 
+    def _upload_extra_args(self) -> Dict[str, Any]:
+        """
+        ExtraArgs for uploads: the native full-object CRC64NVME checksum and an optional public-read ACL.
+
+        :return: ExtraArgs dict
+        """
+        # S3 validates the data against the (SDK computed) checksum before storing
+        extra_args: Dict[str, Any] = {"ChecksumAlgorithm": "CRC64NVME"}
+        if self.public_readable:
+            extra_args["ACL"] = "public-read"
+        return extra_args
+
     def get_s3_transfer_config(self) -> TransferConfig:
         # workaround threading issue https://github.com/boto/s3transfer/issues/197
-        # derived class can overload this if a different config is desired
-        s3_transfer_config = TransferConfig(use_threads=False)
+        # derived class can overload this if a different config is desired (the multipart threshold also determines which native
+        # checksum algorithm uploads use - see _upload_extra_args)
+        s3_transfer_config = TransferConfig(use_threads=False, multipart_threshold=default_multipart_threshold)
         return s3_transfer_config
 
     @typechecked()
@@ -172,7 +212,8 @@ def write_string(self, input_str: str, s3_key: str):
         """
         log.debug(f"writing {self.bucket_name}/{s3_key}")
         assert self.resource is not None
-        self.resource.Object(self.bucket_name, s3_key).put(Body=input_str, Metadata={sha512_string: get_string_sha512(input_str)})
+        # S3 validates the data against the (SDK computed) checksum before storing
+        self.resource.Object(self.bucket_name, s3_key).put(Body=input_str, ChecksumAlgorithm="CRC64NVME")
 
     @typechecked()
     def write_lines(self, input_lines: List[str], s3_key: str):
@@ -212,32 +253,38 @@ def upload(self, file_path: Union[str, Path], s3_key: str, force: bool = False)
             file_path = Path(file_path)
 
         file_mtime = os.path.getmtime(file_path)
-        file_sha512 = get_file_sha512(file_path)
+        file_crc64nvme = get_file_crc64nvme(file_path)
         if force:
             upload_flag = True
         else:
             if self.object_exists(s3_key):
                 s3_object_metadata = self.get_s3_object_metadata(s3_key)
                 log.info(f"{s3_object_metadata=}")
-                if s3_object_metadata.sha512 is not None:
-                    # use the hash provided by awsimple, if it exists (note that .get_sha512() never returns None - it synthesizes a substitute hash - so check .sha512 itself)
-                    upload_flag = file_sha512 != s3_object_metadata.sha512
+                if s3_object_metadata.crc64nvme is not None:
+                    # use the native full-object checksum, if the object has one (note that .get_sha512() never returns None - it synthesizes a substitute hash - so check the field itself)
+                    upload_flag = file_crc64nvme != s3_object_metadata.crc64nvme
+                elif s3_object_metadata.sha512 is not None:
+                    # the object was written by another tool with a native full-object SHA-512 checksum
+                    upload_flag = get_file_sha512(file_path) != s3_object_metadata.sha512
+                elif s3_object_metadata.legacy_sha512 is not None:
+                    # the object was written by an older awsimple using the legacy metadata hash - always re-upload it so it gains the native checksum
+                    upload_flag = True
                 else:
-                    # if not, use mtime
-                    upload_flag = not isclose(file_mtime, s3_object_metadata.mtime.timestamp(), abs_tol=self.mtime_abs_tol)
+                    # no hash is available, so compare modification time and file size (free and robust, but weaker than a hash)
+                    sizes_equal = os.path.getsize(file_path) == s3_object_metadata.size
+                    mtimes_equal = isclose(file_mtime, s3_object_metadata.mtime.timestamp(), abs_tol=self.mtime_abs_tol)
+                    upload_flag = not (sizes_equal and mtimes_equal)
             else:
                 upload_flag = True
 
         uploaded_flag = False
         if upload_flag:
-            log.info(f"local file : {file_sha512=},force={force} - uploading")
+            log.info(f"local file : {file_crc64nvme=},force={force} - uploading")
 
             transfer_retry_count = 0
+            extra_args = self._upload_extra_args()
+            log.info(f"{extra_args=}")
             while not uploaded_flag and transfer_retry_count < self.retry_count:
-                extra_args = {"Metadata": {sha512_string: file_sha512}}
-                if self.public_readable:
-                    extra_args["ACL"] = "public-read"  # type: ignore
-                log.info(f"{extra_args=}")
 
                 try:
                     self.client.upload_file(str(file_path), self.bucket_name, s3_key, ExtraArgs=extra_args, Config=self.get_s3_transfer_config())
@@ -255,7 +302,7 @@ def upload(self, file_path: Union[str, Path], s3_key: str, force: bool = False)
                 raise AWSimpleException(f"couldn't upload {file_path} to {self.bucket_name}/{s3_key} after {self.retry_count} attempts")
 
         else:
-            log.info(f"file hash of {file_sha512} is the same as is already on S3 and force={force} - not uploading")
+            log.info(f"file checksum of {file_crc64nvme} is the same as is already on S3 and force={force} - not uploading")
 
         return uploaded_flag
 
@@ -272,30 +319,33 @@ def upload_object_as_json(self, json_serializable_object: Union[List, Dict], s3_
 
         s3_key = _get_json_key(s3_key)
         json_as_bytes = serializable_object_to_json_as_bytes(json_serializable_object)
-        json_sha512 = get_bytes_sha512(json_as_bytes)
+        json_crc64nvme = get_bytes_crc64nvme(json_as_bytes)
         upload_flag = True
         if not force and self.object_exists(s3_key):
             s3_object_metadata = self.get_s3_object_metadata(s3_key)
             log.info(f"{s3_object_metadata=}")
-            if s3_object_metadata.sha512 is not None:
-                # use the hash provided by awsimple, if it exists (note that .get_sha512() never returns None - it synthesizes a substitute hash - so check .sha512 itself)
-                upload_flag = json_sha512 != s3_object_metadata.sha512
+            if s3_object_metadata.crc64nvme is not None:
+                # use the native full-object checksum, if the object has one (note that .get_sha512() never returns None - it synthesizes a substitute hash - so check the field itself)
+                upload_flag = json_crc64nvme != s3_object_metadata.crc64nvme
+            elif s3_object_metadata.sha512 is not None:
+                # the object was written by another tool with a native full-object SHA-512 checksum
+                upload_flag = get_bytes_sha512(json_as_bytes) != s3_object_metadata.sha512
+            # otherwise upload_flag stays True - objects written by older awsimple (legacy metadata hash) or without any hash are always re-uploaded so they gain the native checksum
 
         uploaded_flag = False
         if upload_flag:
-            log.info(f"{json_sha512=},force={force} - uploading")
+            log.info(f"{json_crc64nvme=},force={force} - uploading")
 
             transfer_retry_count = 0
             while not uploaded_flag and transfer_retry_count < self.retry_count:
-                meta_data = {sha512_string: json_sha512}
-                log.info(f"{meta_data=}")
                 assert self.resource is not None
                 try:
                     s3_object = self.resource.Object(self.bucket_name, s3_key)
+                    # S3 validates the data against the (SDK computed) checksum before storing
+                    put_kwargs: Dict[str, Any] = {"Body": json_as_bytes, "ChecksumAlgorithm": "CRC64NVME"}
                     if self.public_readable:
-                        s3_object.put(Body=json_as_bytes, Metadata=meta_data, ACL="public-read")
-                    else:
-                        s3_object.put(Body=json_as_bytes, Metadata=meta_data)
+                        put_kwargs["ACL"] = "public-read"
+                    s3_object.put(**put_kwargs)
                     uploaded_flag = True
                 except connection_errors as e:
                     log.warning(f"{self.bucket_name}:{s3_key} : {transfer_retry_count=} : {e}")
@@ -306,7 +356,7 @@ def upload_object_as_json(self, json_serializable_object: Union[List, Dict], s3_
                 raise AWSimpleException(f"couldn't upload JSON to {self.bucket_name}/{s3_key} after {self.retry_count} attempts")
 
         else:
-            log.info(f"file hash of {json_sha512} is the same as is already on S3 and force={force} - not uploading")
+            log.info(f"checksum of {json_crc64nvme} is the same as is already on S3 and force={force} - not uploading")
 
         return uploaded_flag
 
@@ -468,20 +518,24 @@ def get_s3_object_metadata(self, s3_key: str) -> S3ObjectMetadata:
         :return: S3ObjectMetadata
         """
         try:
-            head = self.client.head_object(Bucket=self.bucket_name, Key=s3_key)
+            head = self.client.head_object(Bucket=self.bucket_name, Key=s3_key, ChecksumMode="ENABLED")
         except ClientError as e:
             if boto_error_to_string(e) in ("404", "NoSuchKey", "NotFound"):
                 raise AWSimpleException(f"{self.bucket_name=} {s3_key=} does not exist") from e
             raise
         assert isinstance(self.bucket_name, str)  # mainly for mypy
+        # sha512 and crc64nvme are S3's native (server-validated) full-object checksums; legacy_sha512 is awsimple's legacy custom
+        # metadata, kept readable so objects written by older awsimple versions can be detected (and re-uploaded to gain the native checksum)
         s3_object_metadata = S3ObjectMetadata(
             self.bucket_name,
             s3_key,
             head["ContentLength"],
             head["LastModified"],
             head["ETag"][1:-1].lower(),
-            head.get("Metadata", {}).get(sha512_string),
+            _native_checksum_to_hex(head.get("ChecksumSHA512")),
             self.get_s3_object_url(s3_key),
+            head.get("Metadata", {}).get(sha512_string),
+            _native_checksum_to_hex(head.get("ChecksumCRC64NVME")),
         )
         log.debug(f"{s3_object_metadata=}")
         return s3_object_metadata

requirements-dev.txt

@@ -1,7 +1,7 @@
 #
 # awsimple requirements
-hashy
-boto3
+hashy>=0.14.0
+boto3[crt]>=1.43.0
 typeguard
 dictim
 appdirs

setup.py

@@ -23,7 +23,7 @@
     keywords=["aws", "cloud", "storage", "database", "dynamodb", "s3"],
     packages=[__title__],
     package_data={__title__: [readme_file_path, "py.typed"]},
-    install_requires=["boto3", "typeguard", "hashy>=0.1.1", "dictim", "appdirs", "tobool", "urllib3", "python-dateutil", "yasf", "strif"],
+    install_requires=["boto3[crt]>=1.43.0", "typeguard", "hashy>=0.14.0", "dictim", "appdirs", "tobool", "urllib3", "python-dateutil", "yasf", "strif"],  # boto3[crt]>=1.43.0 and hashy>=0.14.0 for CRC64NVME checksum support
     project_urls={"Documentation": "https://awsimple.readthedocs.io/"},
     classifiers=[],
     python_requires=">3.10",

test_awsimple/test_s3_dir.py

@@ -1,6 +1,8 @@
 from pprint import pprint
 from pathlib import Path
 
+from hashy import get_bytes_crc64nvme
+
 from awsimple import S3Access
 
 from test_awsimple import test_awsimple_str, temp_dir
@@ -20,7 +22,7 @@ def test_s3_dir():
     pprint(s3_dir)
     md = s3_dir[test_file_name]
     assert md.key == test_file_name
-    assert md.sha512 == "309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f"  # "hello world"
+    assert md.crc64nvme == get_bytes_crc64nvme(b"hello world")
 
 
 def test_s3_dir_prefix():
@@ -37,4 +39,4 @@ def test_s3_dir_prefix():
     pprint(s3_dir)
     md = s3_dir[test_file_name]
     assert md.key == test_file_name
-    assert md.sha512 == "309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f"  # "hello world"
+    assert md.crc64nvme == get_bytes_crc64nvme(b"hello world")

test_awsimple/test_s3_file_transfer.py

@@ -6,6 +6,8 @@
 from shutil import rmtree
 from logging import getLogger
 
+from hashy import get_bytes_crc64nvme
+
 from awsimple import S3Access, get_directory_size, is_mock, is_using_localstack
 from test_awsimple import test_awsimple_str, never_change_file_name, temp_dir, cache_dir
 
@@ -95,7 +97,7 @@ def test_s3_z_metadata(s3_access):
     test_file_name = "test.txt"
     s3_object_metadata = s3_access.get_s3_object_metadata(test_file_name)
     # "hello world" uploaded with awsimple
-    assert s3_object_metadata.sha512 == "309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f"
+    assert s3_object_metadata.crc64nvme == get_bytes_crc64nvme(b"hello world")
     assert s3_object_metadata.size == 11