From 60e831f14bec6ff87e46132d64a5aaf9d995c28f Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Sun, 8 Feb 2026 08:16:16 +0000 Subject: [PATCH 1/4] Remove the private methods from abstract and async client --- .../_experimental/asyncio/async_client.py | 151 ------------------ google/cloud/storage/abstracts/base_client.py | 84 ---------- tests/unit/asyncio/test_async_client.py | 140 ---------------- 3 files changed, 375 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/async_client.py b/google/cloud/storage/_experimental/asyncio/async_client.py index bd8817a09..1ed24194b 100644 --- a/google/cloud/storage/_experimental/asyncio/async_client.py +++ b/google/cloud/storage/_experimental/asyncio/async_client.py @@ -14,13 +14,6 @@ """Asynchronous client for interacting with Google Cloud Storage.""" -import functools - -from google.cloud.storage._experimental.asyncio.async_helpers import ASYNC_DEFAULT_TIMEOUT -from google.cloud.storage._experimental.asyncio.async_helpers import ASYNC_DEFAULT_RETRY -from google.cloud.storage._experimental.asyncio.async_helpers import AsyncHTTPIterator -from google.cloud.storage._experimental.asyncio.async_helpers import _do_nothing_page_start -from google.cloud.storage._opentelemetry_tracing import create_trace_span from google.cloud.storage._experimental.asyncio.async_creds import AsyncCredsWrapper from google.cloud.storage.abstracts.base_client import BaseClient from google.cloud.storage._experimental.asyncio.async_connection import AsyncConnection @@ -89,150 +82,6 @@ async def close(self): if self._async_http_internal is not None and not self._async_http_passed_by_user: await self._async_http_internal.close() - async def _get_resource( - self, - path, - query_params=None, - headers=None, - timeout=ASYNC_DEFAULT_TIMEOUT, - retry=ASYNC_DEFAULT_RETRY, - _target_object=None, - ): - """See super() class""" - return await self._connection.api_request( - method="GET", - path=path, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=_target_object, - ) - - def _list_resource( - self, - path, - item_to_value, - page_token=None, - max_results=None, - extra_params=None, - page_start=_do_nothing_page_start, - page_size=None, - timeout=ASYNC_DEFAULT_TIMEOUT, - retry=ASYNC_DEFAULT_RETRY, - ): - """See super() class""" - kwargs = { - "method": "GET", - "path": path, - "timeout": timeout, - } - with create_trace_span( - name="Storage.AsyncClient._list_resource_returns_iterator", - client=self, - api_request=kwargs, - retry=retry, - ): - api_request = functools.partial( - self._connection.api_request, timeout=timeout, retry=retry - ) - return AsyncHTTPIterator( - client=self, - api_request=api_request, - path=path, - item_to_value=item_to_value, - page_token=page_token, - max_results=max_results, - extra_params=extra_params, - page_start=page_start, - page_size=page_size, - ) - - async def _patch_resource( - self, - path, - data, - query_params=None, - headers=None, - timeout=ASYNC_DEFAULT_TIMEOUT, - retry=None, - _target_object=None, - ): - """See super() class""" - return await self._connection.api_request( - method="PATCH", - path=path, - data=data, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=_target_object, - ) - - async def _put_resource( - self, - path, - data, - query_params=None, - headers=None, - timeout=ASYNC_DEFAULT_TIMEOUT, - retry=None, - _target_object=None, - ): - """See super() class""" - return await self._connection.api_request( - method="PUT", - path=path, - data=data, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=_target_object, - ) - - async def _post_resource( - self, - path, - data, - query_params=None, - headers=None, - timeout=ASYNC_DEFAULT_TIMEOUT, - retry=None, - _target_object=None, - ): - """See super() class""" - return await self._connection.api_request( - method="POST", - path=path, - data=data, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=_target_object, - ) - - async def _delete_resource( - self, - path, - query_params=None, - headers=None, - timeout=ASYNC_DEFAULT_TIMEOUT, - retry=ASYNC_DEFAULT_RETRY, - _target_object=None, - ): - """See super() class""" - return await self._connection.api_request( - method="DELETE", - path=path, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=_target_object, - ) def bucket(self, bucket_name, user_project=None, generation=None): """Factory constructor for bucket object. diff --git a/google/cloud/storage/abstracts/base_client.py b/google/cloud/storage/abstracts/base_client.py index c2030cb89..7b092e169 100644 --- a/google/cloud/storage/abstracts/base_client.py +++ b/google/cloud/storage/abstracts/base_client.py @@ -299,87 +299,3 @@ def current_batch(self): @abstractmethod def bucket(self, bucket_name, user_project=None, generation=None): raise NotImplementedError("This method needs to be implemented.") - - @abstractmethod - def _get_resource( - self, - path, - query_params=None, - headers=None, - timeout=None, - retry=None, - _target_object=None, - ): - """Helper for bucket / blob methods making API 'GET' calls.""" - raise NotImplementedError("This should be implemented via the child class") - - @abstractmethod - def _list_resource( - self, - path, - item_to_value, - page_token=None, - max_results=None, - extra_params=None, - page_start=None, - page_size=None, - timeout=None, - retry=None, - ): - """Helper for bucket / blob methods making API 'GET' calls.""" - raise NotImplementedError("This should be implemented via the child class") - - @abstractmethod - def _patch_resource( - self, - path, - data, - query_params=None, - headers=None, - timeout=None, - retry=None, - _target_object=None, - ): - """Helper for bucket / blob methods making API 'PATCH' calls.""" - raise NotImplementedError("This should be implemented via the child class") - - @abstractmethod - def _put_resource( - self, - path, - data, - query_params=None, - headers=None, - timeout=None, - retry=None, - _target_object=None, - ): - """Helper for bucket / blob methods making API 'PUT' calls.""" - raise NotImplementedError("This should be implemented via the child class") - - @abstractmethod - def _post_resource( - self, - path, - data, - query_params=None, - headers=None, - timeout=None, - retry=None, - _target_object=None, - ): - """Helper for bucket / blob methods making API 'POST' calls.""" - raise NotImplementedError("This should be implemented via the child class") - - @abstractmethod - def _delete_resource( - self, - path, - query_params=None, - headers=None, - timeout=None, - retry=None, - _target_object=None, - ): - """Helper for bucket / blob methods making API 'DELETE' calls.""" - raise NotImplementedError("This should be implemented via the child class") diff --git a/tests/unit/asyncio/test_async_client.py b/tests/unit/asyncio/test_async_client.py index 64481a0d4..6f005f8ad 100644 --- a/tests/unit/asyncio/test_async_client.py +++ b/tests/unit/asyncio/test_async_client.py @@ -134,146 +134,6 @@ async def test_close_ignores_user_session(self): await client.close() user_session.close.assert_not_awaited() - @pytest.mark.asyncio - async def test_get_resource(self): - path = "/b/bucket" - query_params = {"foo": "bar"} - credentials = _make_credentials() - - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one(project="PROJECT", credentials=credentials) - - # Mock the connection's api_request - client._connection.api_request = mock.AsyncMock( - return_value="response") - - result = await client._get_resource(path, query_params=query_params) - - assert result == "response" - client._connection.api_request.assert_awaited_once_with( - method="GET", - path=path, - query_params=query_params, - headers=None, - timeout=mock.ANY, - retry=mock.ANY, - _target_object=None - ) - - @pytest.mark.asyncio - async def test_list_resource(self): - path = "/b/bucket/o" - item_to_value = mock.Mock() - credentials = _make_credentials() - - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one(project="PROJECT", credentials=credentials) - - iterator = client._list_resource( - path=path, - item_to_value=item_to_value, - max_results=10, - page_token="token" - ) - - assert isinstance(iterator, AsyncHTTPIterator) - assert iterator.path == path - assert iterator.max_results == 10 - - @pytest.mark.asyncio - async def test_patch_resource(self): - path = "/b/bucket" - data = {"key": "val"} - credentials = _make_credentials() - - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one(project="PROJECT", credentials=credentials) - - client._connection.api_request = mock.AsyncMock() - - await client._patch_resource(path, data=data) - - client._connection.api_request.assert_awaited_once_with( - method="PATCH", - path=path, - data=data, - query_params=None, - headers=None, - timeout=mock.ANY, - retry=None, - _target_object=None - ) - - @pytest.mark.asyncio - async def test_put_resource(self): - path = "/b/bucket/o/obj" - data = b"bytes" - credentials = _make_credentials() - - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one(project="PROJECT", credentials=credentials) - - client._connection.api_request = mock.AsyncMock() - - await client._put_resource(path, data=data) - - client._connection.api_request.assert_awaited_once_with( - method="PUT", - path=path, - data=data, - query_params=None, - headers=None, - timeout=mock.ANY, - retry=None, - _target_object=None - ) - - @pytest.mark.asyncio - async def test_post_resource(self): - path = "/b/bucket/o/obj/compose" - data = {"source": []} - credentials = _make_credentials() - - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one(project="PROJECT", credentials=credentials) - - client._connection.api_request = mock.AsyncMock() - - await client._post_resource(path, data=data) - - client._connection.api_request.assert_awaited_once_with( - method="POST", - path=path, - data=data, - query_params=None, - headers=None, - timeout=mock.ANY, - retry=None, - _target_object=None - ) - - @pytest.mark.asyncio - async def test_delete_resource(self): - path = "/b/bucket" - credentials = _make_credentials() - - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one(project="PROJECT", credentials=credentials) - - client._connection.api_request = mock.AsyncMock() - - await client._delete_resource(path) - - client._connection.api_request.assert_awaited_once_with( - method="DELETE", - path=path, - query_params=None, - headers=None, - timeout=mock.ANY, - retry=mock.ANY, - _target_object=None - ) - def test_bucket_not_implemented(self): credentials = _make_credentials() with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): From 019a0d8f8115b660b578c0b0a5ac4c9fff7e19bc Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Sun, 8 Feb 2026 10:08:00 +0000 Subject: [PATCH 2/4] Refactor async_connection class --- .../asyncio/abstracts/__init__.py | 0 .../asyncio/abstracts/async_connection.py | 74 +++++++++ .../_experimental/asyncio/async_client.py | 46 +++--- .../_experimental/asyncio/utility/__init__.py | 0 .../async_json_connection.py} | 88 ++++------ google/cloud/storage/abstracts/base_client.py | 78 --------- google/cloud/storage/client.py | 77 +++++++++ tests/unit/asyncio/test_async_client.py | 80 ++-------- ...ction.py => test_async_json_connection.py} | 151 ++++++++++++------ 9 files changed, 317 insertions(+), 277 deletions(-) create mode 100644 google/cloud/storage/_experimental/asyncio/abstracts/__init__.py create mode 100644 google/cloud/storage/_experimental/asyncio/abstracts/async_connection.py create mode 100644 google/cloud/storage/_experimental/asyncio/utility/__init__.py rename google/cloud/storage/_experimental/asyncio/{async_connection.py => utility/async_json_connection.py} (83%) rename tests/unit/asyncio/{test_async_connection.py => test_async_json_connection.py} (63%) diff --git a/google/cloud/storage/_experimental/asyncio/abstracts/__init__.py b/google/cloud/storage/_experimental/asyncio/abstracts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/google/cloud/storage/_experimental/asyncio/abstracts/async_connection.py b/google/cloud/storage/_experimental/asyncio/abstracts/async_connection.py new file mode 100644 index 000000000..412d5e95f --- /dev/null +++ b/google/cloud/storage/_experimental/asyncio/abstracts/async_connection.py @@ -0,0 +1,74 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abstract class for Async JSON and GRPC connection.""" + +import abc +from google.cloud.storage._http import AGENT_VERSION +from google.api_core.client_info import ClientInfo +from google.cloud.storage import __version__ + + +class AsyncConnection(abc.ABC): + """Class for asynchronous connection with JSON and GRPC compatibility. + + This class expose python implementation of interacting with relevant APIs. + + Args: + client: The client that owns this connection. + client_info: Information about the client library. + """ + + def __init__(self, client, client_info=None): + self._client = client + + if client_info is None: + client_info = ClientInfo() + + self._client_info = client_info + if self._client_info.user_agent is None: + self._client_info.user_agent = AGENT_VERSION + else: + self._client_info.user_agent = ( + f"{self._client_info.user_agent} {AGENT_VERSION}" + ) + self._client_info.client_library_version = __version__ + self._extra_headers = {} + + @property + def extra_headers(self): + """Returns extra headers to send with every request.""" + return self._extra_headers + + @extra_headers.setter + def extra_headers(self, value): + """Set the extra header property.""" + self._extra_headers = value + + @property + def user_agent(self): + """Returns user_agent for async HTTP transport. + + Returns: + str: The user agent string. + """ + return self._client_info.to_user_agent() + + @user_agent.setter + def user_agent(self, value): + """Setter for user_agent in connection.""" + self._client_info.user_agent = value + + async def close(self): + pass \ No newline at end of file diff --git a/google/cloud/storage/_experimental/asyncio/async_client.py b/google/cloud/storage/_experimental/asyncio/async_client.py index 1ed24194b..b60c0d611 100644 --- a/google/cloud/storage/_experimental/asyncio/async_client.py +++ b/google/cloud/storage/_experimental/asyncio/async_client.py @@ -16,16 +16,9 @@ from google.cloud.storage._experimental.asyncio.async_creds import AsyncCredsWrapper from google.cloud.storage.abstracts.base_client import BaseClient -from google.cloud.storage._experimental.asyncio.async_connection import AsyncConnection +from google.cloud.storage._experimental.asyncio.utility.async_json_connection import AsyncJSONConnection from google.cloud.storage.abstracts import base_client -try: - from google.auth.aio.transport import sessions - AsyncSession = sessions.AsyncAuthorizedSession - _AIO_AVAILABLE = True -except ImportError: - _AIO_AVAILABLE = False - _marker = base_client.marker @@ -43,13 +36,6 @@ def __init__( *, api_key=None, ): - if not _AIO_AVAILABLE: - # Python 3.9 or less comes with an older version of google-auth library which doesn't support asyncio - raise ImportError( - "Failed to import 'google.auth.aio', Consider using a newer python version (>=3.10)" - " or newer version of google-auth library to mitigate this issue." - ) - if self._use_client_cert: # google.auth.aio.transports.sessions.AsyncAuthorizedSession currently doesn't support configuring mTLS. # In future, we can monkey patch the above, and do provide mTLS support, but that is not a priority @@ -66,22 +52,28 @@ def __init__( api_key=api_key ) self.credentials = AsyncCredsWrapper(self._credentials) # self._credential is synchronous. - self._connection = AsyncConnection(self, **self.connection_kw_args) # adapter for async communication - self._async_http_internal = _async_http - self._async_http_passed_by_user = (_async_http is not None) + self._async_http = _async_http + + # We need both, as the same client can be used for multiple buckets. + self._json_connection_internal = None + self._grpc_connection_internal = None @property - def async_http(self): - """Returns the existing asynchronous session, or create one if it does not exists.""" - if self._async_http_internal is None: - self._async_http_internal = AsyncSession(credentials=self.credentials) - return self._async_http_internal + def _grpc_connection(self): + raise NotImplementedError("Not yet Implemented.") - async def close(self): - """Close the session, if it exists""" - if self._async_http_internal is not None and not self._async_http_passed_by_user: - await self._async_http_internal.close() + @property + def _json_connection(self): + if not self._json_connection_internal: + self._json_connection_internal = AsyncJSONConnection(self, _async_http=self._async_http, credentials=self.credentials, **self.connection_kw_args) + return self._json_connection_internal + async def close(self): + if self._json_connection_internal: + await self._json_connection_internal.close() + + if self._grpc_connection_internal: + await self._grpc_connection_internal.close() def bucket(self, bucket_name, user_project=None, generation=None): """Factory constructor for bucket object. diff --git a/google/cloud/storage/_experimental/asyncio/utility/__init__.py b/google/cloud/storage/_experimental/asyncio/utility/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/google/cloud/storage/_experimental/asyncio/async_connection.py b/google/cloud/storage/_experimental/asyncio/utility/async_json_connection.py similarity index 83% rename from google/cloud/storage/_experimental/asyncio/async_connection.py rename to google/cloud/storage/_experimental/asyncio/utility/async_json_connection.py index 8d297ef8a..f9769633a 100644 --- a/google/cloud/storage/_experimental/asyncio/async_connection.py +++ b/google/cloud/storage/_experimental/asyncio/utility/async_json_connection.py @@ -12,28 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Create/interact with Google Cloud Storage connections in asynchronous manner.""" +"""Implementation of Async JSON connection""" import json import collections import functools -from urllib.parse import urlencode - import google.api_core.exceptions + +from urllib.parse import urlencode from google.cloud import _http from google.cloud.storage import _http as storage_http from google.cloud.storage import _helpers -from google.api_core.client_info import ClientInfo from google.cloud.storage._opentelemetry_tracing import create_trace_span -from google.cloud.storage import __version__ -from google.cloud.storage._http import AGENT_VERSION - +from google.cloud.storage._experimental.asyncio.abstracts.async_connection import AsyncConnection -class AsyncConnection: - """Class for asynchronous connection using google.auth.aio. +try: + from google.auth.aio.transport import sessions + AsyncSession = sessions.AsyncAuthorizedSession + _AIO_AVAILABLE = True +except ImportError: + _AIO_AVAILABLE = False - This class handles the creation of API requests, header management, - user agent configuration, and error handling for the Async Storage Client. +class AsyncJSONConnection(AsyncConnection): + """Implementation of Async JSON connection Args: client: The client that owns this connection. @@ -41,58 +42,35 @@ class AsyncConnection: api_endpoint: The API endpoint to use. """ - def __init__(self, client, client_info=None, api_endpoint=None): - self._client = client - - if client_info is None: - client_info = ClientInfo() - - self._client_info = client_info - if self._client_info.user_agent is None: - self._client_info.user_agent = AGENT_VERSION - else: - self._client_info.user_agent = ( - f"{self._client_info.user_agent} {AGENT_VERSION}" + def __init__(self, client, client_info=None, api_endpoint=None, _async_http=None, credentials=None): + if not _AIO_AVAILABLE: + # Python 3.9 or less comes with an older version of google-auth library which doesn't support asyncio + raise ImportError( + "Failed to import 'google.auth.aio', Consider using a newer python version (>=3.10)" + " or newer version of google-auth library to mitigate this issue." ) - self._client_info.client_library_version = __version__ - self._extra_headers = {} + + super().__init__(client, client_info=client_info) self.API_BASE_URL = api_endpoint or storage_http.Connection.DEFAULT_API_ENDPOINT self.API_VERSION = storage_http.Connection.API_VERSION self.API_URL_TEMPLATE = storage_http.Connection.API_URL_TEMPLATE - - @property - def extra_headers(self): - """Returns extra headers to send with every request.""" - return self._extra_headers - - @extra_headers.setter - def extra_headers(self, value): - """Set the extra header property.""" - self._extra_headers = value + + self.credentials = credentials + self._async_http_internal = _async_http + self._async_http_passed_by_user = (_async_http is not None) @property def async_http(self): - """Returns the AsyncAuthorizedSession from the client. - - Returns: - google.auth.aio.transport.sessions.AsyncAuthorizedSession: The async session. - """ - return self._client.async_http - - @property - def user_agent(self): - """Returns user_agent for async HTTP transport. - - Returns: - str: The user agent string. - """ - return self._client_info.to_user_agent() - - @user_agent.setter - def user_agent(self, value): - """Setter for user_agent in connection.""" - self._client_info.user_agent = value + """Returns the existing asynchronous session, or create one if it does not exists.""" + if self._async_http_internal is None: + self._async_http_internal = AsyncSession(credentials=self.credentials) + return self._async_http_internal + + async def close(self): + """Close the session, if it exists""" + if self._async_http_internal is not None and not self._async_http_passed_by_user: + await self._async_http_internal.close() async def _make_request( self, diff --git a/google/cloud/storage/abstracts/base_client.py b/google/cloud/storage/abstracts/base_client.py index 7b092e169..e0ed2e3da 100644 --- a/google/cloud/storage/abstracts/base_client.py +++ b/google/cloud/storage/abstracts/base_client.py @@ -203,53 +203,6 @@ def create_anonymous_client(cls): client.project = None return client - @property - def api_endpoint(self): - """Returns the API_BASE_URL from connection""" - return self._connection.API_BASE_URL - - def update_user_agent(self, user_agent): - """Update the user-agent string for this client. - - :type user_agent: str - :param user_agent: The string to add to the user-agent. - """ - existing_user_agent = self._connection._client_info.user_agent - if existing_user_agent is None: - self._connection.user_agent = user_agent - else: - self._connection.user_agent = f"{user_agent} {existing_user_agent}" - - @property - def _connection(self): - """Get connection or batch on the client. - - :rtype: :class:`google.cloud.storage._http.Connection` - :returns: The connection set on the client, or the batch - if one is set. - """ - if self.current_batch is not None: - return self.current_batch - else: - return self._base_connection - - @_connection.setter - def _connection(self, value): - """Set connection on the client. - - Intended to be used by constructor (since the base class calls) - self._connection = connection - Will raise if the connection is set more than once. - - :type value: :class:`google.cloud.storage._http.Connection` - :param value: The connection set on the client. - - :raises: :class:`ValueError` if connection has already been set. - """ - if self._base_connection is not None: - raise ValueError("Connection already set on client") - self._base_connection = value - @property def _use_client_cert(self): """Returns true if mTLS is enabled""" @@ -265,37 +218,6 @@ def _use_client_cert(self): ) return use_client_cert - def _push_batch(self, batch): - """Push a batch onto our stack. - - "Protected", intended for use by batch context mgrs. - - :type batch: :class:`google.cloud.storage.batch.Batch` - :param batch: newly-active batch - """ - self._batch_stack.push(batch) - - def _pop_batch(self): - """Pop a batch from our stack. - - "Protected", intended for use by batch context mgrs. - - :raises: IndexError if the stack is empty. - :rtype: :class:`google.cloud.storage.batch.Batch` - :returns: the top-most batch/transaction, after removing it. - """ - return self._batch_stack.pop() - - @property - def current_batch(self): - """Currently-active batch. - - :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if - no batch is active). - :returns: The batch at the top of the batch stack. - """ - return self._batch_stack.top - @abstractmethod def bucket(self, bucket_name, user_project=None, generation=None): raise NotImplementedError("This method needs to be implemented.") diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index afa0b3a4a..c1f1661d0 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -147,6 +147,83 @@ def __init__( connection.extra_headers = extra_headers self._connection = connection + @property + def api_endpoint(self): + """Returns the API_BASE_URL from connection""" + return self._connection.API_BASE_URL + + def update_user_agent(self, user_agent): + """Update the user-agent string for this client. + + :type user_agent: str + :param user_agent: The string to add to the user-agent. + """ + existing_user_agent = self._connection._client_info.user_agent + if existing_user_agent is None: + self._connection.user_agent = user_agent + else: + self._connection.user_agent = f"{user_agent} {existing_user_agent}" + + @property + def _connection(self): + """Get connection or batch on the client. + + :rtype: :class:`google.cloud.storage._http.Connection` + :returns: The connection set on the client, or the batch + if one is set. + """ + if self.current_batch is not None: + return self.current_batch + else: + return self._base_connection + + @_connection.setter + def _connection(self, value): + """Set connection on the client. + + Intended to be used by constructor (since the base class calls) + self._connection = connection + Will raise if the connection is set more than once. + + :type value: :class:`google.cloud.storage._http.Connection` + :param value: The connection set on the client. + + :raises: :class:`ValueError` if connection has already been set. + """ + if self._base_connection is not None: + raise ValueError("Connection already set on client") + self._base_connection = value + + def _push_batch(self, batch): + """Push a batch onto our stack. + + "Protected", intended for use by batch context mgrs. + + :type batch: :class:`google.cloud.storage.batch.Batch` + :param batch: newly-active batch + """ + self._batch_stack.push(batch) + + def _pop_batch(self): + """Pop a batch from our stack. + + "Protected", intended for use by batch context mgrs. + + :raises: IndexError if the stack is empty. + :rtype: :class:`google.cloud.storage.batch.Batch` + :returns: the top-most batch/transaction, after removing it. + """ + return self._batch_stack.pop() + + @property + def current_batch(self): + """Currently-active batch. + + :rtype: :class:`google.cloud.storage.batch.Batch` or ``NoneType`` (if + no batch is active). + :returns: The batch at the top of the batch stack. + """ + return self._batch_stack.top def get_service_account_email( self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY diff --git a/tests/unit/asyncio/test_async_client.py b/tests/unit/asyncio/test_async_client.py index 6f005f8ad..735daa660 100644 --- a/tests/unit/asyncio/test_async_client.py +++ b/tests/unit/asyncio/test_async_client.py @@ -17,10 +17,6 @@ import pytest from google.auth.credentials import Credentials from google.cloud.storage._experimental.asyncio.async_client import AsyncClient -from google.cloud.storage._experimental.asyncio.async_helpers import AsyncHTTPIterator - -# Aliases to match sync test style -_marker = object() def _make_credentials(): @@ -45,22 +41,16 @@ def test_ctor_defaults(self): PROJECT = "PROJECT" credentials = _make_credentials() - # We mock AsyncConnection to prevent network logic during init - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection") as MockConn: + # We mock AsyncJSONConnection to prevent network logic during init + with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection") as MockConn: client = self._make_one(project=PROJECT, credentials=credentials) - assert client.project == PROJECT - # It is the instance of the Mock - assert isinstance(client._connection, mock.Mock) - assert client._connection == MockConn.return_value - - # Verify specific async attributes - assert client._async_http_internal is None - assert client._async_http_passed_by_user is False + assert client.project == PROJECT - # Verify inheritance from BaseClient worked (batch stack, etc) - assert client.current_batch is None - assert list(client._batch_stack) == [] + # It is the instance of the Mock + assert isinstance(client._json_connection, mock.Mock) + assert client._json_connection == MockConn.return_value + MockConn.assert_called_once_with(client, _async_http=None, credentials=client.credentials, client_info=None, api_endpoint=None) def test_ctor_mtls_raises_error(self): credentials = _make_credentials() @@ -76,67 +66,19 @@ def test_ctor_w_async_http_passed(self): credentials = _make_credentials() async_http = mock.Mock() - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): + with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection") as MockConn: client = self._make_one( project="PROJECT", credentials=credentials, _async_http=async_http ) - assert client._async_http_internal is async_http - assert client._async_http_passed_by_user is True - - def test_async_http_property_creates_session(self): - credentials = _make_credentials() - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one(project="PROJECT", credentials=credentials) - - assert client._async_http_internal is None - - # Mock the auth session class - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncSession") as MockSession: - session = client.async_http - - assert session is MockSession.return_value - assert client._async_http_internal is session - # Should be initialized with the AsyncCredsWrapper, not the raw credentials - MockSession.assert_called_once() - call_kwargs = MockSession.call_args[1] - assert call_kwargs['credentials'] == client.credentials - - @pytest.mark.asyncio - async def test_close_manages_session_lifecycle(self): - credentials = _make_credentials() - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one(project="PROJECT", credentials=credentials) - - # 1. Internal session created by client -> Client closes it - mock_internal = mock.AsyncMock() - client._async_http_internal = mock_internal - client._async_http_passed_by_user = False - - await client.close() - mock_internal.close.assert_awaited_once() - - @pytest.mark.asyncio - async def test_close_ignores_user_session(self): - credentials = _make_credentials() - user_session = mock.AsyncMock() - - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): - client = self._make_one( - project="PROJECT", - credentials=credentials, - _async_http=user_session - ) - - # 2. External session passed by user -> Client DOES NOT close it - await client.close() - user_session.close.assert_not_awaited() + client._json_connection + MockConn.assert_called_once_with(client, _async_http=async_http, credentials=client.credentials, client_info=None, api_endpoint=None) def test_bucket_not_implemented(self): credentials = _make_credentials() - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncConnection"): + with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection"): client = self._make_one(project="PROJECT", credentials=credentials) with pytest.raises(NotImplementedError): diff --git a/tests/unit/asyncio/test_async_connection.py b/tests/unit/asyncio/test_async_json_connection.py similarity index 63% rename from tests/unit/asyncio/test_async_connection.py rename to tests/unit/asyncio/test_async_json_connection.py index 5a4dde8c2..2f564e17f 100644 --- a/tests/unit/asyncio/test_async_connection.py +++ b/tests/unit/asyncio/test_async_json_connection.py @@ -1,11 +1,18 @@ -import json +import sys import pytest +import json from unittest import mock - from google.cloud.storage import _http as storage_http from google.api_core import exceptions +from google.auth.credentials import Credentials from google.api_core.client_info import ClientInfo -from google.cloud.storage._experimental.asyncio.async_connection import AsyncConnection +from google.cloud.storage._experimental.asyncio.utility.async_json_connection import ( + AsyncJSONConnection, +) + +pytestmark = pytest.mark.skipif( + sys.version_info[:2] == (3, 9), reason="Not supported on 3.9" +) class MockAuthResponse: @@ -24,26 +31,22 @@ async def read(self): return self._data -@pytest.fixture -def mock_client(): - """Mocks the Google Cloud Storage Client.""" - client = mock.Mock() - client.async_http = mock.AsyncMock() - return client +def _make_credentials(): + creds = mock.Mock(spec=Credentials) + creds.universe_domain = "googleapis.com" + return creds @pytest.fixture -def async_connection(mock_client): - """Creates an instance of AsyncConnection with a mocked client.""" - return AsyncConnection(mock_client) +def async_connection(): + """Creates an instance of AsyncJSONConnection with a mocked client.""" + return AsyncJSONConnection(mock.Mock(), _async_http=mock.AsyncMock()) @pytest.fixture def mock_trace_span(): """Mocks the OpenTelemetry trace span context manager.""" - target = ( - "google.cloud.storage._experimental.asyncio.async_connection.create_trace_span" - ) + target = "google.cloud.storage._experimental.asyncio.utility.async_json_connection.create_trace_span" with mock.patch(target) as mock_span: mock_span.return_value.__enter__.return_value = None yield mock_span @@ -58,13 +61,75 @@ def test_init_defaults(async_connection): assert "gcloud-python" in async_connection.user_agent -def test_init_custom_endpoint(mock_client): +def test_init_custom_endpoint(): """Test initialization with a custom API endpoint.""" custom_endpoint = "https://custom.storage.googleapis.com" - conn = AsyncConnection(mock_client, api_endpoint=custom_endpoint) + conn = AsyncJSONConnection(mock.Mock(), api_endpoint=custom_endpoint) assert conn.API_BASE_URL == custom_endpoint +def test_passing_async_http_uses_the_same_async_http(): + async_http = mock.Mock() + conn = AsyncJSONConnection(mock.Mock(), _async_http=async_http) + assert conn.async_http == async_http + + +@pytest.mark.asyncio +async def test_not_passing_async_http_creates_new(): + with mock.patch( + "google.cloud.storage._experimental.asyncio.utility.async_json_connection.AsyncSession" + ) as MockSession: + conn = AsyncJSONConnection( + mock.Mock(), _async_http=None, credentials=_make_credentials() + ) + assert conn.async_http is MockSession.return_value + + +@pytest.mark.asyncio +async def test_close_async_json_connection(): + with mock.patch( + "google.cloud.storage._experimental.asyncio.utility.async_json_connection.AsyncSession" + ): + conn = AsyncJSONConnection( + mock.Mock(), _async_http=None, credentials=_make_credentials() + ) + + internal_mock = mock.AsyncMock() + conn._async_http_internal = internal_mock + await conn.close() + internal_mock.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_close_manages_internal_session(): + """Tests that close() is called on an internally created session.""" + with mock.patch( + "google.cloud.storage._experimental.asyncio.utility.async_json_connection.AsyncSession" + ): + conn = AsyncJSONConnection( + mock.Mock(), _async_http=None, credentials=_make_credentials() + ) + + # Force session creation + session = conn.async_http + session.close = mock.AsyncMock() # Make it an async mock + + await conn.close() + session.close.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_close_ignores_user_session(): + """Tests that close() is NOT called on a user-provided session.""" + user_session = mock.AsyncMock() + conn = AsyncJSONConnection( + mock.Mock(), _async_http=user_session, credentials=_make_credentials() + ) + + await conn.close() + user_session.close.assert_not_awaited() + + def test_extra_headers_property(async_connection): """Test getter and setter for extra_headers.""" headers = {"X-Custom-Header": "value"} @@ -92,10 +157,10 @@ def test_build_api_url_with_params(async_connection): @pytest.mark.asyncio -async def test_make_request_headers(async_connection, mock_client): +async def test_make_request_headers(async_connection): """Test that _make_request adds the correct headers.""" mock_response = MockAuthResponse(status_code=200) - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response async_connection.user_agent = "test-agent/1.0" async_connection.extra_headers = {"X-Test": "True"} @@ -104,7 +169,7 @@ async def test_make_request_headers(async_connection, mock_client): method="GET", url="http://example.com", content_type="application/json" ) - call_args = mock_client.async_http.request.call_args + call_args = async_connection.async_http.request.call_args _, kwargs = call_args headers = kwargs["headers"] @@ -117,13 +182,13 @@ async def test_make_request_headers(async_connection, mock_client): @pytest.mark.asyncio -async def test_api_request_success(async_connection, mock_client, mock_trace_span): +async def test_api_request_success(async_connection, mock_trace_span): """Test the high-level api_request method wraps the call correctly.""" expected_data = {"items": []} mock_response = MockAuthResponse( status_code=200, data=json.dumps(expected_data).encode("utf-8") ) - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response response = await async_connection.api_request(method="GET", path="/b/bucket") @@ -132,17 +197,15 @@ async def test_api_request_success(async_connection, mock_client, mock_trace_spa @pytest.mark.asyncio -async def test_perform_api_request_json_serialization( - async_connection, mock_client, mock_trace_span -): +async def test_perform_api_request_json_serialization(async_connection): """Test that dictionary data is serialized to JSON.""" mock_response = MockAuthResponse(status_code=200) - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response data = {"key": "value"} await async_connection.api_request(method="POST", path="/b", data=data) - call_args = mock_client.async_http.request.call_args + call_args = async_connection.async_http.request.call_args _, kwargs = call_args assert kwargs["data"] == json.dumps(data) @@ -150,15 +213,13 @@ async def test_perform_api_request_json_serialization( @pytest.mark.asyncio -async def test_perform_api_request_error_handling( - async_connection, mock_client, mock_trace_span -): +async def test_perform_api_request_error_handling(async_connection): """Test that non-2xx responses raise GoogleAPICallError.""" error_json = {"error": {"message": "Not Found"}} mock_response = MockAuthResponse( status_code=404, data=json.dumps(error_json).encode("utf-8") ) - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response with pytest.raises(exceptions.GoogleAPICallError) as excinfo: await async_connection.api_request(method="GET", path="/b/nonexistent") @@ -167,13 +228,11 @@ async def test_perform_api_request_error_handling( @pytest.mark.asyncio -async def test_perform_api_request_no_json_response( - async_connection, mock_client, mock_trace_span -): +async def test_perform_api_request_no_json_response(async_connection): """Test response handling when expect_json is False.""" raw_bytes = b"binary_data" mock_response = MockAuthResponse(status_code=200, data=raw_bytes) - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response response = await async_connection.api_request( method="GET", path="/b/obj", expect_json=False @@ -183,10 +242,10 @@ async def test_perform_api_request_no_json_response( @pytest.mark.asyncio -async def test_api_request_with_retry(async_connection, mock_client, mock_trace_span): +async def test_api_request_with_retry(async_connection): """Test that the retry policy is applied if conditions are met.""" mock_response = MockAuthResponse(status_code=200, data=b"{}") - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response mock_retry = mock.Mock() mock_policy = mock.Mock(side_effect=lambda call: call) @@ -217,12 +276,10 @@ def test_build_api_url_overrides(async_connection): @pytest.mark.asyncio -async def test_perform_api_request_empty_response( - async_connection, mock_client, mock_trace_span -): +async def test_perform_api_request_empty_response(async_connection): """Test handling of empty 2xx response when expecting JSON.""" mock_response = MockAuthResponse(status_code=204, data=b"") - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response response = await async_connection.api_request( method="DELETE", path="/b/bucket/o/object" @@ -232,13 +289,11 @@ async def test_perform_api_request_empty_response( @pytest.mark.asyncio -async def test_perform_api_request_non_json_error( - async_connection, mock_client, mock_trace_span -): +async def test_perform_api_request_non_json_error(async_connection): """Test error handling when the error response is plain text (not JSON).""" error_text = "Bad Gateway" mock_response = MockAuthResponse(status_code=502, data=error_text.encode("utf-8")) - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response with pytest.raises(exceptions.GoogleAPICallError) as excinfo: await async_connection.api_request(method="GET", path="/b/bucket") @@ -248,10 +303,10 @@ async def test_perform_api_request_non_json_error( @pytest.mark.asyncio -async def test_make_request_extra_api_info(async_connection, mock_client): +async def test_make_request_extra_api_info(async_connection): """Test logic for constructing x-goog-api-client header with extra info.""" mock_response = MockAuthResponse(status_code=200) - mock_client.async_http.request.return_value = mock_response + async_connection.async_http.request.return_value = mock_response invocation_id = "test-id-123" @@ -259,7 +314,7 @@ async def test_make_request_extra_api_info(async_connection, mock_client): method="GET", url="http://example.com", extra_api_info=invocation_id ) - call_args = mock_client.async_http.request.call_args + call_args = async_connection.async_http.request.call_args _, kwargs = call_args headers = kwargs["headers"] From 4a20fc681c8fcf38878488f453611975d4efc841 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Mon, 9 Feb 2026 11:24:26 +0000 Subject: [PATCH 3/4] lint changes --- .../asyncio/abstracts/async_connection.py | 4 +- .../_experimental/asyncio/async_client.py | 19 ++++-- .../_experimental/asyncio/async_creds.py | 10 ++-- .../_experimental/asyncio/async_helpers.py | 9 ++- .../asyncio/utility/async_json_connection.py | 24 ++++++-- .../cloud/storage/_media/requests/download.py | 1 - google/cloud/storage/abstracts/base_client.py | 5 +- google/cloud/storage/client.py | 7 ++- tests/unit/asyncio/test_async_client.py | 43 ++++++++++---- tests/unit/asyncio/test_async_creds.py | 41 +++++++------ tests/unit/asyncio/test_async_helpers.py | 59 +++++++++---------- tests/unit/test_blob.py | 8 ++- 12 files changed, 139 insertions(+), 91 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/abstracts/async_connection.py b/google/cloud/storage/_experimental/asyncio/abstracts/async_connection.py index 412d5e95f..6ed843a5b 100644 --- a/google/cloud/storage/_experimental/asyncio/abstracts/async_connection.py +++ b/google/cloud/storage/_experimental/asyncio/abstracts/async_connection.py @@ -45,7 +45,7 @@ def __init__(self, client, client_info=None): ) self._client_info.client_library_version = __version__ self._extra_headers = {} - + @property def extra_headers(self): """Returns extra headers to send with every request.""" @@ -71,4 +71,4 @@ def user_agent(self, value): self._client_info.user_agent = value async def close(self): - pass \ No newline at end of file + pass diff --git a/google/cloud/storage/_experimental/asyncio/async_client.py b/google/cloud/storage/_experimental/asyncio/async_client.py index b60c0d611..c24b0d948 100644 --- a/google/cloud/storage/_experimental/asyncio/async_client.py +++ b/google/cloud/storage/_experimental/asyncio/async_client.py @@ -16,7 +16,9 @@ from google.cloud.storage._experimental.asyncio.async_creds import AsyncCredsWrapper from google.cloud.storage.abstracts.base_client import BaseClient -from google.cloud.storage._experimental.asyncio.utility.async_json_connection import AsyncJSONConnection +from google.cloud.storage._experimental.asyncio.utility.async_json_connection import ( + AsyncJSONConnection, +) from google.cloud.storage.abstracts import base_client _marker = base_client.marker @@ -49,9 +51,11 @@ def __init__( client_info=client_info, client_options=client_options, extra_headers=extra_headers, - api_key=api_key + api_key=api_key, ) - self.credentials = AsyncCredsWrapper(self._credentials) # self._credential is synchronous. + self.credentials = AsyncCredsWrapper( + self._credentials + ) # self._credential is synchronous. self._async_http = _async_http # We need both, as the same client can be used for multiple buckets. @@ -65,13 +69,18 @@ def _grpc_connection(self): @property def _json_connection(self): if not self._json_connection_internal: - self._json_connection_internal = AsyncJSONConnection(self, _async_http=self._async_http, credentials=self.credentials, **self.connection_kw_args) + self._json_connection_internal = AsyncJSONConnection( + self, + _async_http=self._async_http, + credentials=self.credentials, + **self.connection_kw_args, + ) return self._json_connection_internal async def close(self): if self._json_connection_internal: await self._json_connection_internal.close() - + if self._grpc_connection_internal: await self._grpc_connection_internal.close() diff --git a/google/cloud/storage/_experimental/asyncio/async_creds.py b/google/cloud/storage/_experimental/asyncio/async_creds.py index 2fb899b19..e2abc3316 100644 --- a/google/cloud/storage/_experimental/asyncio/async_creds.py +++ b/google/cloud/storage/_experimental/asyncio/async_creds.py @@ -5,21 +5,23 @@ try: from google.auth.aio import credentials as aio_creds_module + BaseCredentials = aio_creds_module.Credentials _AIO_AVAILABLE = True except ImportError: BaseCredentials = object _AIO_AVAILABLE = False + class AsyncCredsWrapper(BaseCredentials): """Wraps synchronous Google Auth credentials to provide an asynchronous interface. Args: - sync_creds (google.auth.credentials.Credentials): The synchronous credentials + sync_creds (google.auth.credentials.Credentials): The synchronous credentials instance to wrap. Raises: - ImportError: If instantiated in an environment where 'google.auth.aio' + ImportError: If instantiated in an environment where 'google.auth.aio' is not available. """ @@ -36,9 +38,7 @@ def __init__(self, sync_creds): async def refresh(self, request): """Refreshes the access token.""" loop = asyncio.get_running_loop() - await loop.run_in_executor( - None, self.creds.refresh, Request() - ) + await loop.run_in_executor(None, self.creds.refresh, Request()) @property def valid(self): diff --git a/google/cloud/storage/_experimental/asyncio/async_helpers.py b/google/cloud/storage/_experimental/asyncio/async_helpers.py index bfebfaafa..4a7d78732 100644 --- a/google/cloud/storage/_experimental/asyncio/async_helpers.py +++ b/google/cloud/storage/_experimental/asyncio/async_helpers.py @@ -24,6 +24,7 @@ async def _do_nothing_page_start(iterator, page, response): # pylint: disable=unused-argument pass + class AsyncHTTPIterator(AsyncIterator): """A generic class for iterating through HTTP/JSON API list responses asynchronously. @@ -32,7 +33,7 @@ class AsyncHTTPIterator(AsyncIterator): api_request (Callable): The **async** function to use to make API requests. This must be an awaitable. path (str): The method path to query for the list of items. - item_to_value (Callable[AsyncIterator, Any]): Callable to convert an item + item_to_value (Callable[AsyncIterator, Any]): Callable to convert an item from the type in the JSON response into a native object. items_key (str): The key in the API response where the list of items can be found. @@ -40,7 +41,7 @@ class AsyncHTTPIterator(AsyncIterator): page_size (int): The maximum number of results to fetch per page. max_results (int): The maximum number of results to fetch. extra_params (dict): Extra query string parameters for the API call. - page_start (Callable): Callable to provide special behavior after a new page + page_start (Callable): Callable to provide special behavior after a new page is created. next_token (str): The name of the field used in the response for page tokens. """ @@ -137,6 +138,4 @@ def _get_query_params(self): async def _get_next_page_response(self): """Requests the next page from the path provided asynchronously.""" params = self._get_query_params() - return await self.api_request( - method="GET", path=self.path, query_params=params - ) + return await self.api_request(method="GET", path=self.path, query_params=params) diff --git a/google/cloud/storage/_experimental/asyncio/utility/async_json_connection.py b/google/cloud/storage/_experimental/asyncio/utility/async_json_connection.py index f9769633a..3b8fc79b6 100644 --- a/google/cloud/storage/_experimental/asyncio/utility/async_json_connection.py +++ b/google/cloud/storage/_experimental/asyncio/utility/async_json_connection.py @@ -24,15 +24,19 @@ from google.cloud.storage import _http as storage_http from google.cloud.storage import _helpers from google.cloud.storage._opentelemetry_tracing import create_trace_span -from google.cloud.storage._experimental.asyncio.abstracts.async_connection import AsyncConnection +from google.cloud.storage._experimental.asyncio.abstracts.async_connection import ( + AsyncConnection, +) try: from google.auth.aio.transport import sessions + AsyncSession = sessions.AsyncAuthorizedSession _AIO_AVAILABLE = True except ImportError: _AIO_AVAILABLE = False + class AsyncJSONConnection(AsyncConnection): """Implementation of Async JSON connection @@ -42,7 +46,14 @@ class AsyncJSONConnection(AsyncConnection): api_endpoint: The API endpoint to use. """ - def __init__(self, client, client_info=None, api_endpoint=None, _async_http=None, credentials=None): + def __init__( + self, + client, + client_info=None, + api_endpoint=None, + _async_http=None, + credentials=None, + ): if not _AIO_AVAILABLE: # Python 3.9 or less comes with an older version of google-auth library which doesn't support asyncio raise ImportError( @@ -55,10 +66,10 @@ def __init__(self, client, client_info=None, api_endpoint=None, _async_http=None self.API_BASE_URL = api_endpoint or storage_http.Connection.DEFAULT_API_ENDPOINT self.API_VERSION = storage_http.Connection.API_VERSION self.API_URL_TEMPLATE = storage_http.Connection.API_URL_TEMPLATE - + self.credentials = credentials self._async_http_internal = _async_http - self._async_http_passed_by_user = (_async_http is not None) + self._async_http_passed_by_user = _async_http is not None @property def async_http(self): @@ -69,7 +80,10 @@ def async_http(self): async def close(self): """Close the session, if it exists""" - if self._async_http_internal is not None and not self._async_http_passed_by_user: + if ( + self._async_http_internal is not None + and not self._async_http_passed_by_user + ): await self._async_http_internal.close() async def _make_request( diff --git a/google/cloud/storage/_media/requests/download.py b/google/cloud/storage/_media/requests/download.py index 13e049bd3..c5686fcb7 100644 --- a/google/cloud/storage/_media/requests/download.py +++ b/google/cloud/storage/_media/requests/download.py @@ -774,6 +774,5 @@ def flush(self): def has_unconsumed_tail(self) -> bool: return self._decoder.has_unconsumed_tail - else: # pragma: NO COVER _BrotliDecoder = None # type: ignore # pragma: NO COVER diff --git a/google/cloud/storage/abstracts/base_client.py b/google/cloud/storage/abstracts/base_client.py index e0ed2e3da..e1b7d2971 100644 --- a/google/cloud/storage/abstracts/base_client.py +++ b/google/cloud/storage/abstracts/base_client.py @@ -30,6 +30,7 @@ marker = object() + class BaseClient(ClientWithProject, ABC): """Abstract class for python-storage Client""" @@ -213,9 +214,7 @@ def _use_client_cert(self): if hasattr(mtls, "should_use_client_cert"): use_client_cert = mtls.should_use_client_cert() else: - use_client_cert = ( - os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true" - ) + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true" return use_client_cert @abstractmethod diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index c1f1661d0..b46c34485 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -50,6 +50,7 @@ _marker = base_client.marker + def _buckets_page_start(iterator, page, response): """Grab unreachable buckets after a :class:`~google.cloud.iterator.Page` started.""" unreachable = response.get("unreachable", []) @@ -139,11 +140,13 @@ def __init__( client_options=client_options, use_auth_w_custom_endpoint=use_auth_w_custom_endpoint, extra_headers=extra_headers, - api_key=api_key + api_key=api_key, ) # Pass extra_headers to Connection - connection = Connection(self, **self.connection_kw_args) # connection_kw_args would always be set in base class + connection = Connection( + self, **self.connection_kw_args + ) # connection_kw_args would always be set in base class connection.extra_headers = extra_headers self._connection = connection diff --git a/tests/unit/asyncio/test_async_client.py b/tests/unit/asyncio/test_async_client.py index 735daa660..57f9a8369 100644 --- a/tests/unit/asyncio/test_async_client.py +++ b/tests/unit/asyncio/test_async_client.py @@ -27,7 +27,7 @@ def _make_credentials(): @pytest.mark.skipif( sys.version_info < (3, 10), - reason="Async Client requires Python 3.10+ due to google-auth-library asyncio support" + reason="Async Client requires Python 3.10+ due to google-auth-library asyncio support", ) class TestAsyncClient: @staticmethod @@ -42,7 +42,9 @@ def test_ctor_defaults(self): credentials = _make_credentials() # We mock AsyncJSONConnection to prevent network logic during init - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection") as MockConn: + with mock.patch( + "google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection" + ) as MockConn: client = self._make_one(project=PROJECT, credentials=credentials) assert client.project == PROJECT @@ -50,35 +52,54 @@ def test_ctor_defaults(self): # It is the instance of the Mock assert isinstance(client._json_connection, mock.Mock) assert client._json_connection == MockConn.return_value - MockConn.assert_called_once_with(client, _async_http=None, credentials=client.credentials, client_info=None, api_endpoint=None) + MockConn.assert_called_once_with( + client, + _async_http=None, + credentials=client.credentials, + client_info=None, + api_endpoint=None, + ) def test_ctor_mtls_raises_error(self): credentials = _make_credentials() # Simulate environment where mTLS is enabled - with mock.patch("google.cloud.storage.abstracts.base_client.BaseClient._use_client_cert", new_callable=mock.PropertyMock) as mock_mtls: + with mock.patch( + "google.cloud.storage.abstracts.base_client.BaseClient._use_client_cert", + new_callable=mock.PropertyMock, + ) as mock_mtls: mock_mtls.return_value = True - with pytest.raises(ValueError, match="Async Client currently do not support mTLS"): + with pytest.raises( + ValueError, match="Async Client currently do not support mTLS" + ): self._make_one(credentials=credentials) def test_ctor_w_async_http_passed(self): credentials = _make_credentials() async_http = mock.Mock() - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection") as MockConn: + with mock.patch( + "google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection" + ) as MockConn: client = self._make_one( - project="PROJECT", - credentials=credentials, - _async_http=async_http + project="PROJECT", credentials=credentials, _async_http=async_http ) client._json_connection - MockConn.assert_called_once_with(client, _async_http=async_http, credentials=client.credentials, client_info=None, api_endpoint=None) + MockConn.assert_called_once_with( + client, + _async_http=async_http, + credentials=client.credentials, + client_info=None, + api_endpoint=None, + ) def test_bucket_not_implemented(self): credentials = _make_credentials() - with mock.patch("google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection"): + with mock.patch( + "google.cloud.storage._experimental.asyncio.async_client.AsyncJSONConnection" + ): client = self._make_one(project="PROJECT", credentials=credentials) with pytest.raises(NotImplementedError): diff --git a/tests/unit/asyncio/test_async_creds.py b/tests/unit/asyncio/test_async_creds.py index 0a45bca5d..3dad11fd0 100644 --- a/tests/unit/asyncio/test_async_creds.py +++ b/tests/unit/asyncio/test_async_creds.py @@ -4,28 +4,30 @@ from google.auth import credentials as google_creds from google.cloud.storage._experimental.asyncio import async_creds + @pytest.fixture def mock_aio_modules(): """Patches sys.modules to simulate google.auth.aio existence.""" mock_creds_module = unittest.mock.MagicMock() # We must set the base class to object so our wrapper can inherit safely in tests - mock_creds_module.Credentials = object - + mock_creds_module.Credentials = object + modules = { - 'google.auth.aio': unittest.mock.MagicMock(), - 'google.auth.aio.credentials': mock_creds_module, + "google.auth.aio": unittest.mock.MagicMock(), + "google.auth.aio.credentials": mock_creds_module, } - + with unittest.mock.patch.dict(sys.modules, modules): # We also need to manually flip the flag in the module to True for the test context # because the module was likely already imported with the flag set to False/True # depending on the real environment. - with unittest.mock.patch.object(async_creds, '_AIO_AVAILABLE', True): + with unittest.mock.patch.object(async_creds, "_AIO_AVAILABLE", True): # We also need to ensure BaseCredentials in the module points to our mock # if we want strictly correct inheritance, though duck typing usually suffices. - with unittest.mock.patch.object(async_creds, 'BaseCredentials', object): + with unittest.mock.patch.object(async_creds, "BaseCredentials", object): yield + @pytest.fixture def mock_sync_creds(): """Creates a mock of the synchronous Google Credentials object.""" @@ -33,14 +35,15 @@ def mock_sync_creds(): type(creds).valid = unittest.mock.PropertyMock(return_value=True) return creds + @pytest.fixture def async_wrapper(mock_aio_modules, mock_sync_creds): """Instantiates the wrapper with the mock credentials.""" # This instantiation would raise ImportError if mock_aio_modules didn't set _AIO_AVAILABLE=True return async_creds.AsyncCredsWrapper(mock_sync_creds) + class TestAsyncCredsWrapper: - @pytest.mark.asyncio async def test_init_sets_attributes(self, async_wrapper, mock_sync_creds): """Test that the wrapper initializes correctly.""" @@ -51,19 +54,19 @@ async def test_valid_property_delegates(self, async_wrapper, mock_sync_creds): """Test that the .valid property maps to the sync creds .valid property.""" type(mock_sync_creds).valid = unittest.mock.PropertyMock(return_value=True) assert async_wrapper.valid is True - + type(mock_sync_creds).valid = unittest.mock.PropertyMock(return_value=False) assert async_wrapper.valid is False @pytest.mark.asyncio async def test_refresh_offloads_to_executor(self, async_wrapper, mock_sync_creds): - """Test that refresh() gets the running loop and calls sync refresh in executor.""" - with unittest.mock.patch('asyncio.get_running_loop') as mock_get_loop: + """Test that refresh() gets the running loop and calls sync refresh in executor.""" + with unittest.mock.patch("asyncio.get_running_loop") as mock_get_loop: mock_loop = unittest.mock.AsyncMock() mock_get_loop.return_value = mock_loop - + await async_wrapper.refresh(None) - + mock_loop.run_in_executor.assert_called_once() args, _ = mock_loop.run_in_executor.call_args assert args[1] == mock_sync_creds.refresh @@ -72,10 +75,10 @@ async def test_refresh_offloads_to_executor(self, async_wrapper, mock_sync_creds async def test_before_request_valid_creds(self, async_wrapper, mock_sync_creds): """Test before_request when credentials are ALREADY valid.""" type(mock_sync_creds).valid = unittest.mock.PropertyMock(return_value=True) - + headers = {} await async_wrapper.before_request(None, "GET", "http://example.com", headers) - + mock_sync_creds.apply.assert_called_once_with(headers) mock_sync_creds.before_request.assert_not_called() @@ -83,12 +86,12 @@ async def test_before_request_valid_creds(self, async_wrapper, mock_sync_creds): async def test_before_request_invalid_creds(self, async_wrapper, mock_sync_creds): """Test before_request when credentials are INVALID (refresh path).""" type(mock_sync_creds).valid = unittest.mock.PropertyMock(return_value=False) - + headers = {} method = "GET" url = "http://example.com" - with unittest.mock.patch('asyncio.get_running_loop') as mock_get_loop: + with unittest.mock.patch("asyncio.get_running_loop") as mock_get_loop: mock_loop = unittest.mock.AsyncMock() mock_get_loop.return_value = mock_loop @@ -101,8 +104,8 @@ async def test_before_request_invalid_creds(self, async_wrapper, mock_sync_creds def test_missing_aio_raises_error(self, mock_sync_creds): """Ensure ImportError is raised if _AIO_AVAILABLE is False.""" # We manually simulate the environment where AIO is missing - with unittest.mock.patch.object(async_creds, '_AIO_AVAILABLE', False): + with unittest.mock.patch.object(async_creds, "_AIO_AVAILABLE", False): with pytest.raises(ImportError) as excinfo: async_creds.AsyncCredsWrapper(mock_sync_creds) - + assert "Failed to import 'google.auth.aio'" in str(excinfo.value) diff --git a/tests/unit/asyncio/test_async_helpers.py b/tests/unit/asyncio/test_async_helpers.py index 58ebbea31..d125f2b57 100644 --- a/tests/unit/asyncio/test_async_helpers.py +++ b/tests/unit/asyncio/test_async_helpers.py @@ -27,7 +27,6 @@ async def _safe_anext(iterator): class TestAsyncHTTPIterator: - def _make_one(self, *args, **kw): return AsyncHTTPIterator(*args, **kw) @@ -35,11 +34,9 @@ def _make_one(self, *args, **kw): async def test_iterate_items_single_page(self): """Test simple iteration over one page of results.""" client = mock.Mock() - api_request = mock.AsyncMock() - api_request.return_value = { - "items": ["a", "b"] - } - + api_request = mock.AsyncMock() + api_request.return_value = {"items": ["a", "b"]} + iterator = self._make_one( client=client, api_request=api_request, @@ -53,11 +50,9 @@ async def test_iterate_items_single_page(self): assert results == ["A", "B"] assert iterator.num_results == 2 - assert iterator.page_number == 1 + assert iterator.page_number == 1 api_request.assert_awaited_once_with( - method="GET", - path="/path", - query_params={} + method="GET", path="/path", query_params={} ) @pytest.mark.asyncio @@ -65,14 +60,14 @@ async def test_iterate_items_multiple_pages(self): """Test pagination flow passes tokens correctly.""" client = mock.Mock() api_request = mock.AsyncMock() - + # Setup Response: 2 Pages api_request.side_effect = [ - {"items": ["1", "2"], "nextPageToken": "token-A"}, # Page 1 - {"items": ["3"], "nextPageToken": "token-B"}, # Page 2 - {"items": []} # Page 3 (Empty/End) + {"items": ["1", "2"], "nextPageToken": "token-A"}, # Page 1 + {"items": ["3"], "nextPageToken": "token-B"}, # Page 2 + {"items": []}, # Page 3 (Empty/End) ] - + iterator = self._make_one( client=client, api_request=api_request, @@ -84,7 +79,7 @@ async def test_iterate_items_multiple_pages(self): assert results == [1, 2, 3] assert api_request.call_count == 3 - + calls = api_request.call_args_list assert calls[0].kwargs["query_params"] == {} assert calls[1].kwargs["query_params"] == {"pageToken": "token-A"} @@ -95,12 +90,12 @@ async def test_iterate_pages_public_property(self): """Test the .pages property which yields Page objects instead of items.""" client = mock.Mock() api_request = mock.AsyncMock() - + api_request.side_effect = [ {"items": ["a"], "nextPageToken": "next"}, - {"items": ["b"]} + {"items": ["b"]}, ] - + iterator = self._make_one( client=client, api_request=api_request, @@ -115,7 +110,7 @@ async def test_iterate_pages_public_property(self): assert len(pages) == 2 assert list(pages[0]) == ["a"] - assert list(pages[1]) == ["b"] + assert list(pages[1]) == ["b"] assert iterator.page_number == 2 @pytest.mark.asyncio @@ -123,7 +118,7 @@ async def test_max_results_limits_requests(self): """Test that max_results alters the request parameters dynamically.""" client = mock.Mock() api_request = mock.AsyncMock() - + # Setup: We want 5 items total. # Page 1 returns 3 items. # Page 2 *should* only be asked for 2 items. @@ -131,24 +126,24 @@ async def test_max_results_limits_requests(self): {"items": ["a", "b", "c"], "nextPageToken": "t1"}, {"items": ["d", "e"], "nextPageToken": "t2"}, ] - + iterator = self._make_one( client=client, api_request=api_request, path="/path", item_to_value=lambda _, x: x, - max_results=5 # <--- Limit set here + max_results=5, # <--- Limit set here ) results = [i async for i in iterator] assert len(results) == 5 assert results == ["a", "b", "c", "d", "e"] - + # Verify Request 1: Asked for max 5 call1_params = api_request.call_args_list[0].kwargs["query_params"] assert call1_params["maxResults"] == 5 - + # Verify Request 2: Asked for max 2 (5 - 3 already fetched) call2_params = api_request.call_args_list[1].kwargs["query_params"] assert call2_params["maxResults"] == 2 @@ -159,15 +154,15 @@ async def test_extra_params_passthrough(self): """Test that extra_params are merged into every request.""" client = mock.Mock() api_request = mock.AsyncMock(return_value={"items": []}) - + custom_params = {"projection": "full", "delimiter": "/"} - + iterator = self._make_one( client=client, api_request=api_request, path="/path", item_to_value=mock.Mock(), - extra_params=custom_params # <--- Input + extra_params=custom_params, # <--- Input ) # Trigger a request @@ -183,13 +178,13 @@ async def test_page_size_configuration(self): """Test that page_size is sent as maxResults if no global max_results is set.""" client = mock.Mock() api_request = mock.AsyncMock(return_value={"items": []}) - + iterator = self._make_one( client=client, api_request=api_request, path="/path", item_to_value=mock.Mock(), - page_size=50 # <--- User preference + page_size=50, # <--- User preference ) await _safe_anext(iterator) @@ -210,7 +205,7 @@ async def test_page_start_callback(self): api_request=api_request, path="/path", item_to_value=lambda _, x: x, - page_start=callback + page_start=callback, ) # Run iteration @@ -258,7 +253,7 @@ async def test_error_if_iterated_twice(self): # First Start async for _ in iterator: pass - + # Second Start (Should Fail) with pytest.raises(ValueError, match="Iterator has already started"): async for _ in iterator: diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index cbf53b398..a8abb1571 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -3064,7 +3064,13 @@ def _make_resumable_transport( fake_response2 = self._mock_requests_response( http.client.PERMANENT_REDIRECT, headers2 ) - json_body = json.dumps({"size": str(total_bytes), "md5Hash": md5_checksum_value, "crc32c": crc32c_checksum_value}) + json_body = json.dumps( + { + "size": str(total_bytes), + "md5Hash": md5_checksum_value, + "crc32c": crc32c_checksum_value, + } + ) if data_corruption: fake_response3 = DataCorruption(None) else: From e311c78693648bc5f4afcf90520e9b0904a73919 Mon Sep 17 00:00:00 2001 From: Chandra Shekhar Sirimala Date: Wed, 11 Feb 2026 20:32:57 +0530 Subject: [PATCH 4/4] chore: remove python 3.9 support. (#1748) chore: remove python 3.9 support. Details in b/483015736 --- .github/sync-repo-settings.yaml | 2 +- .kokoro/presubmit/{system-3.9.cfg => system-3.10.cfg} | 2 +- .librarian/generator-input/noxfile.py | 7 ++----- .librarian/generator-input/setup.py | 3 --- noxfile.py | 7 +------ setup.py | 5 +---- 6 files changed, 6 insertions(+), 20 deletions(-) rename .kokoro/presubmit/{system-3.9.cfg => system-3.10.cfg} (91%) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 19c1d0ba4..073e7d995 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -10,5 +10,5 @@ branchProtectionRules: - 'Kokoro' - 'cla/google' - 'Kokoro system-3.14' - - 'Kokoro system-3.9' + - 'Kokoro system-3.10' - 'OwlBot Post Processor' diff --git a/.kokoro/presubmit/system-3.9.cfg b/.kokoro/presubmit/system-3.10.cfg similarity index 91% rename from .kokoro/presubmit/system-3.9.cfg rename to .kokoro/presubmit/system-3.10.cfg index d21467d02..26958ac2a 100644 --- a/.kokoro/presubmit/system-3.9.cfg +++ b/.kokoro/presubmit/system-3.10.cfg @@ -3,7 +3,7 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "system-3.9" + value: "system-3.10" } # Credentials needed to test universe domain. diff --git a/.librarian/generator-input/noxfile.py b/.librarian/generator-input/noxfile.py index ca527decd..c9ada0739 100644 --- a/.librarian/generator-input/noxfile.py +++ b/.librarian/generator-input/noxfile.py @@ -27,8 +27,8 @@ BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.14" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.14"] -UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.10", "3.14"] +UNIT_TEST_PYTHON_VERSIONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.12"] CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -44,9 +44,6 @@ "lint", "lint_setup_py", "system", - # TODO(https://github.com/googleapis/python-storage/issues/1499): - # Remove or restore testing for Python 3.7/3.8 - "unit-3.9", "unit-3.10", "unit-3.11", "unit-3.12", diff --git a/.librarian/generator-input/setup.py b/.librarian/generator-input/setup.py index 89971aa33..294e63892 100644 --- a/.librarian/generator-input/setup.py +++ b/.librarian/generator-input/setup.py @@ -87,9 +87,6 @@ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", diff --git a/noxfile.py b/noxfile.py index 14dfb29d0..4c2b70193 100644 --- a/noxfile.py +++ b/noxfile.py @@ -17,7 +17,6 @@ from __future__ import absolute_import import os import pathlib -import re import shutil import nox @@ -27,9 +26,8 @@ BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] DEFAULT_PYTHON_VERSION = "3.14" -SYSTEM_TEST_PYTHON_VERSIONS = ["3.9", "3.14"] +SYSTEM_TEST_PYTHON_VERSIONS = ["3.10", "3.14"] UNIT_TEST_PYTHON_VERSIONS = [ - "3.9", "3.10", "3.11", "3.12", @@ -51,9 +49,6 @@ "lint", "lint_setup_py", "system", - # TODO(https://github.com/googleapis/python-storage/issues/1499): - # Remove or restore testing for Python 3.7/3.8 - "unit-3.9", "unit-3.10", "unit-3.11", "unit-3.12", diff --git a/setup.py b/setup.py index b45053856..d3215cff6 100644 --- a/setup.py +++ b/setup.py @@ -99,9 +99,6 @@ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -114,7 +111,7 @@ packages=packages, install_requires=dependencies, extras_require=extras, - python_requires=">=3.7", + python_requires=">=3.10", include_package_data=True, zip_safe=False, )