diff --git a/.librarian/state.yaml b/.librarian/state.yaml
index c863eb88ee08..0c448204e92f 100644
--- a/.librarian/state.yaml
+++ b/.librarian/state.yaml
@@ -3690,6 +3690,15 @@ libraries:
remove_regex:
- packages/google-maps-solar
tag_format: '{id}-v{version}'
+ - id: google-resumable-media
+ version: 2.8.0
+ last_generated_commit: ""
+ apis: []
+ source_roots:
+ - packages/google-resumable-media
+ preserve_regex: []
+ remove_regex: []
+ tag_format: '{id}-v{version}'
- id: google-shopping-css
version: 0.3.0
last_generated_commit: 3322511885371d2b2253f209ccc3aa60d4100cfd
diff --git a/owl-bot-staging/google-resumable-media/google-resumable-media/google-resumable-media.txt b/owl-bot-staging/google-resumable-media/google-resumable-media/google-resumable-media.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-resumable-media/.coveragerc b/packages/google-resumable-media/.coveragerc
new file mode 100644
index 000000000000..8ded7e0f107d
--- /dev/null
+++ b/packages/google-resumable-media/.coveragerc
@@ -0,0 +1,9 @@
+[run]
+branch = True
+
+[report]
+fail_under = 100
+show_missing = True
+exclude_lines =
+ # Re-enable the standard pragma
+ pragma: NO COVER
diff --git a/packages/google-resumable-media/.flake8 b/packages/google-resumable-media/.flake8
new file mode 100644
index 000000000000..c2a44dc56f46
--- /dev/null
+++ b/packages/google-resumable-media/.flake8
@@ -0,0 +1,7 @@
+[flake8]
+ignore = E203, E266, E501, W503
+exclude =
+ __pycache__,
+ .git,
+ *.pyc,
+ conf.py
diff --git a/packages/google-resumable-media/.repo-metadata.json b/packages/google-resumable-media/.repo-metadata.json
new file mode 100644
index 000000000000..1e4c926f061e
--- /dev/null
+++ b/packages/google-resumable-media/.repo-metadata.json
@@ -0,0 +1,12 @@
+{
+ "name": "google-resumable-media",
+ "name_pretty": "Google Resumable Media",
+ "client_documentation": "https://cloud.google.com/python/docs/reference/google-resumable-media/latest",
+ "release_level": "preview",
+ "language": "python",
+ "library_type": "CORE",
+ "repo": "googleapis/google-resumable-media-python",
+ "distribution_name": "google-resumable-media",
+ "default_version": "",
+ "codeowner_team": "@googleapis/gcs-team"
+}
diff --git a/packages/google-resumable-media/CHANGELOG.md b/packages/google-resumable-media/CHANGELOG.md
new file mode 100644
index 000000000000..3f62a6ea0a5e
--- /dev/null
+++ b/packages/google-resumable-media/CHANGELOG.md
@@ -0,0 +1,534 @@
+# Changelog
+
+[PyPI History][1]
+
+[1]: https://pypi.org/project/google-resumable-media/#history
+
+## [2.8.0](https://github.com/googleapis/google-cloud-python/compare/google-resumable-media-v2.7.2...google-resumable-media-v2.8.0) (2025-11-11)
+
+
+### Features
+
+* Add support for Python 3.13 and 3.14 (#485) ([9937233ded26924179d717b69df7c76c93f8c133](https://github.com/googleapis/google-cloud-python/commit/9937233ded26924179d717b69df7c76c93f8c133))
+
+
+### Bug Fixes
+
+* remove setup.cfg configuration for creating universal wheels (#484) ([75dbecf8d140483da27dffc970e2e87338e71432](https://github.com/googleapis/google-cloud-python/commit/75dbecf8d140483da27dffc970e2e87338e71432))
+* resolve issue where pre-release versions of dependencies are installed (#481) ([23dafcf3f216a090a0d0c6941048c7da13cbe496](https://github.com/googleapis/google-cloud-python/commit/23dafcf3f216a090a0d0c6941048c7da13cbe496))
+
+## [2.7.2](https://github.com/googleapis/google-resumable-media-python/compare/v2.7.1...v2.7.2) (2024-08-07)
+
+
+### Bug Fixes
+
+* Correctly calculate starting offset for retries of ranged reads ([#450](https://github.com/googleapis/google-resumable-media-python/issues/450)) ([34302b4](https://github.com/googleapis/google-resumable-media-python/commit/34302b4eb984330cc057ddebe3e9ff55a2745b01))
+
+## [2.7.1](https://github.com/googleapis/google-resumable-media-python/compare/v2.7.0...v2.7.1) (2024-05-31)
+
+
+### Bug Fixes
+
+* Add a check for partial response data ([#435](https://github.com/googleapis/google-resumable-media-python/issues/435)) ([aaea392](https://github.com/googleapis/google-resumable-media-python/commit/aaea3921cb4caacf3cb142639bad41dfaf7ebb28))
+
+## [2.7.0](https://github.com/googleapis/google-resumable-media-python/compare/v2.6.0...v2.7.0) (2023-12-10)
+
+
+### Features
+
+* Add support for Python 3.12 ([#407](https://github.com/googleapis/google-resumable-media-python/issues/407)) ([4f90013](https://github.com/googleapis/google-resumable-media-python/commit/4f900139dd9f5d158342e24f9d8657ab0134a4d8))
+* Support brotli encoding ([#403](https://github.com/googleapis/google-resumable-media-python/issues/403)) ([295e40a](https://github.com/googleapis/google-resumable-media-python/commit/295e40ae414b1a6372796f3831e192fe174e45ee))
+
+## [2.6.0](https://github.com/googleapis/google-resumable-media-python/compare/v2.5.0...v2.6.0) (2023-09-06)
+
+
+### Features
+
+* Add support for concurrent XML MPU uploads ([#395](https://github.com/googleapis/google-resumable-media-python/issues/395)) ([a8d56bc](https://github.com/googleapis/google-resumable-media-python/commit/a8d56bc7f51b9b75f5735b241b4bcdb5b2c62fd6))
+* Introduce compatibility with native namespace packages ([#385](https://github.com/googleapis/google-resumable-media-python/issues/385)) ([cdd7a5e](https://github.com/googleapis/google-resumable-media-python/commit/cdd7a5ec140d9e4f77fa7201d9d6dd289f82dabf))
+
+
+### Bug Fixes
+
+* Add google-auth to aiohttp extra ([#386](https://github.com/googleapis/google-resumable-media-python/issues/386)) ([30c2ebd](https://github.com/googleapis/google-resumable-media-python/commit/30c2ebdb18be28cae522df9fd2af45f1cb2b406d))
+
+## [2.5.0](https://github.com/googleapis/google-resumable-media-python/compare/v2.4.1...v2.5.0) (2023-04-21)
+
+
+### Features
+
+* Add support to retry known connection errors ([#375](https://github.com/googleapis/google-resumable-media-python/issues/375)) ([147e845](https://github.com/googleapis/google-resumable-media-python/commit/147e8458578cc500dfe98d9ac7a447332fb9c52b))
+
+## [2.4.1](https://github.com/googleapis/google-resumable-media-python/compare/v2.4.0...v2.4.1) (2023-01-06)
+
+
+### Bug Fixes
+
+* Avoid validating checksums for partial responses ([#361](https://github.com/googleapis/google-resumable-media-python/issues/361)) ([354287f](https://github.com/googleapis/google-resumable-media-python/commit/354287f023c34e269ca5ef0b24b28b9c37ae9dd7))
+
+## [2.4.0](https://github.com/googleapis/google-resumable-media-python/compare/v2.3.3...v2.4.0) (2022-09-29)
+
+
+### Features
+
+* Handle interrupted downloads with decompressive transcoding ([#346](https://github.com/googleapis/google-resumable-media-python/issues/346)) ([f4d26b7](https://github.com/googleapis/google-resumable-media-python/commit/f4d26b7317bf452c8bc4e7f140f9d10e088b8644))
+
+
+### Bug Fixes
+
+* Allow recover to check the status of upload regardless of state ([#343](https://github.com/googleapis/google-resumable-media-python/issues/343)) ([3599267](https://github.com/googleapis/google-resumable-media-python/commit/3599267df25e54be8d1aa07a673f74d7230aa0b7))
+* Require python 3.7+ ([#337](https://github.com/googleapis/google-resumable-media-python/issues/337)) ([942665f](https://github.com/googleapis/google-resumable-media-python/commit/942665f1bb01d2efb604e0be52736690160973b9))
+* Use unittest.mock ([#329](https://github.com/googleapis/google-resumable-media-python/issues/329)) ([82f9769](https://github.com/googleapis/google-resumable-media-python/commit/82f9769f3368404d1854dd22eeed34eeb25ea835))
+
+
+### Documentation
+
+* Fix changelog header to consistent size ([#331](https://github.com/googleapis/google-resumable-media-python/issues/331)) ([7b1dc9c](https://github.com/googleapis/google-resumable-media-python/commit/7b1dc9cc547d6cff7d1340d5b688d1cb0c492e2a))
+
+## [2.3.3](https://github.com/googleapis/google-resumable-media-python/compare/v2.3.2...v2.3.3) (2022-05-05)
+
+
+### Bug Fixes
+
+* retry client side requests timeout ([#319](https://github.com/googleapis/google-resumable-media-python/issues/319)) ([d0649c7](https://github.com/googleapis/google-resumable-media-python/commit/d0649c7509f4a45623d8676cbc37690864e1ca2f))
+
+## [2.3.2](https://github.com/googleapis/google-resumable-media-python/compare/v2.3.1...v2.3.2) (2022-03-08)
+
+
+### Bug Fixes
+
+* append existing headers in prepare_initiate_request ([#314](https://github.com/googleapis/google-resumable-media-python/issues/314)) ([dfaa317](https://github.com/googleapis/google-resumable-media-python/commit/dfaa31703b1bdce80012622687f8cb02db7f4570))
+
+## [2.3.1](https://github.com/googleapis/google-resumable-media-python/compare/v2.3.0...v2.3.1) (2022-03-03)
+
+
+### Bug Fixes
+
+* include existing headers in prepare request ([#309](https://github.com/googleapis/google-resumable-media-python/issues/309)) ([010680b](https://github.com/googleapis/google-resumable-media-python/commit/010680b942365bb8bcfd326015a3d99a9f0ec825))
+
+## [2.3.0](https://github.com/googleapis/google-resumable-media-python/compare/v2.2.1...v2.3.0) (2022-02-11)
+
+
+### Features
+
+* safely resume interrupted downloads ([#294](https://github.com/googleapis/google-resumable-media-python/issues/294)) ([b363329](https://github.com/googleapis/google-resumable-media-python/commit/b36332915a783ef748bc6f8126bc6b41ee9a044d))
+
+## [2.2.1](https://github.com/googleapis/google-resumable-media-python/compare/v2.2.0...v2.2.1) (2022-02-09)
+
+
+### Bug Fixes
+
+* don't overwrite user-agent on requests ([42b380e](https://github.com/googleapis/google-resumable-media-python/commit/42b380e9ec7aba59aa205f8f354764c9b7e35f19))
+
+## [2.2.0](https://github.com/googleapis/google-resumable-media-python/compare/v2.1.0...v2.2.0) (2022-01-28)
+
+
+### Features
+
+* add 'py.typed' declaration file ([#287](https://github.com/googleapis/google-resumable-media-python/issues/287)) ([cee4164](https://github.com/googleapis/google-resumable-media-python/commit/cee416449701b72e7fd532585a1f739b02b6ab32))
+* add support for signed resumable upload URLs ([#290](https://github.com/googleapis/google-resumable-media-python/issues/290)) ([e1290f5](https://github.com/googleapis/google-resumable-media-python/commit/e1290f523808c7ef5be7dd335a5c94cd1739e6e3))
+
+
+### Bug Fixes
+
+* add user-agent on requests ([#295](https://github.com/googleapis/google-resumable-media-python/issues/295)) ([e107a0c](https://github.com/googleapis/google-resumable-media-python/commit/e107a0cac7ca367015a025a6872a8ad28c7ff15c))
+
+## [2.1.0](https://www.github.com/googleapis/google-resumable-media-python/compare/v2.0.3...v2.1.0) (2021-10-20)
+
+
+### Features
+
+* add support for Python 3.10 ([#279](https://www.github.com/googleapis/google-resumable-media-python/issues/279)) ([4dbd14a](https://www.github.com/googleapis/google-resumable-media-python/commit/4dbd14aed14b87d4d288584a59e8ea11beccaf97))
+
+
+### Bug Fixes
+
+* Include ConnectionError and urllib3 exception as retriable ([#282](https://www.github.com/googleapis/google-resumable-media-python/issues/282)) ([d33465f](https://www.github.com/googleapis/google-resumable-media-python/commit/d33465fc047f4188dd967871ea93255aefd4ac2e))
+
+## [2.0.3](https://www.github.com/googleapis/google-resumable-media-python/compare/v2.0.2...v2.0.3) (2021-09-20)
+
+
+### Bug Fixes
+
+* add REQUEST_TIMEOUT 408 as retryable code ([#270](https://www.github.com/googleapis/google-resumable-media-python/issues/270)) ([d0ad0aa](https://www.github.com/googleapis/google-resumable-media-python/commit/d0ad0aade5f4e7c8efed4f4339fc31fb3304fd3c))
+* un-pin google-crc32c ([#267](https://www.github.com/googleapis/google-resumable-media-python/issues/267)) ([6b03a13](https://www.github.com/googleapis/google-resumable-media-python/commit/6b03a13717e1d4d18186bdf2146d5b452d9e3237))
+
+## [2.0.2](https://www.github.com/googleapis/google-resumable-media-python/compare/v2.0.1...v2.0.2) (2021-09-02)
+
+
+### Bug Fixes
+
+* temporarily pin google-crc32c to 1.1.2 to mitigate upstream issue affecting OS X Big Sur ([#264](https://www.github.com/googleapis/google-resumable-media-python/issues/264)) ([9fa344f](https://www.github.com/googleapis/google-resumable-media-python/commit/9fa344f42a99db1af27b8ca126a2ea6b3c01d837))
+
+## [2.0.1](https://www.github.com/googleapis/google-resumable-media-python/compare/v2.0.0...v2.0.1) (2021-08-30)
+
+
+### Bug Fixes
+
+* check if retry is allowed after retry wait calculation ([#258](https://www.github.com/googleapis/google-resumable-media-python/issues/258)) ([00ccf71](https://www.github.com/googleapis/google-resumable-media-python/commit/00ccf7120251d3899c8d0c2eccdf3b177b5b3742))
+* do not mark upload download instances invalid with retriable error codes ([#261](https://www.github.com/googleapis/google-resumable-media-python/issues/261)) ([a1c5f7d](https://www.github.com/googleapis/google-resumable-media-python/commit/a1c5f7d0e3ce48d8d6eb8aced31707a881f7ee96))
+
+## [2.0.0](https://www.github.com/googleapis/google-resumable-media-python/compare/v2.0.0-b1...v2.0.0) (2021-08-19)
+
+
+### ⚠ BREAKING CHANGES
+
+* drop Python 2.7 support ([#229](https://www.github.com/googleapis/google-resumable-media-python/issues/229)) ([af10d4b](https://www.github.com/googleapis/google-resumable-media-python/commit/af10d4b9a5a3f97f08cf1c634f13b0fb24fc83b3))
+
+### Bug Fixes
+
+* retry ConnectionError and similar errors that occur mid-download ([#251](https://www.github.com/googleapis/google-resumable-media-python/issues/251)) ([bb3ec13](https://www.github.com/googleapis/google-resumable-media-python/commit/bb3ec13f5dbc0e26795cebce957247ecbb525f7b))
+
+## [2.0.0b1](https://www.github.com/googleapis/google-resumable-media-python/compare/v1.3.3...v2.0.0b1) (2021-08-02)
+
+
+### ⚠ BREAKING CHANGES
+
+* drop Python 2.7 support ([#229](https://www.github.com/googleapis/google-resumable-media-python/issues/229)) ([af10d4b](https://www.github.com/googleapis/google-resumable-media-python/commit/af10d4b9a5a3f97f08cf1c634f13b0fb24fc83b3))
+
+## [1.3.3](https://www.github.com/googleapis/google-resumable-media-python/compare/v1.3.2...v1.3.3) (2021-07-30)
+
+
+### Reverts
+
+* revert "fix: add retry coverage to the streaming portion of a download" ([#245](https://www.github.com/googleapis/google-resumable-media-python/issues/245)) ([98673d0](https://www.github.com/googleapis/google-resumable-media-python/commit/98673d01e90de8ea8fb101348dd9d15ae4e0531d))
+
+## [1.3.2](https://www.github.com/googleapis/google-resumable-media-python/compare/v1.3.1...v1.3.2) (2021-07-27)
+
+
+### Bug Fixes
+
+* add retry coverage to the streaming portion of a download ([#241](https://www.github.com/googleapis/google-resumable-media-python/issues/241)) ([cc1f07c](https://www.github.com/googleapis/google-resumable-media-python/commit/cc1f07c241876dba62927f841b1a61aa2554996a))
+
+## [1.3.1](https://www.github.com/googleapis/google-resumable-media-python/compare/v1.3.0...v1.3.1) (2021-06-18)
+
+
+### Bug Fixes
+
+* **deps:** require six>=1.4.0 ([#194](https://www.github.com/googleapis/google-resumable-media-python/issues/194)) ([a840691](https://www.github.com/googleapis/google-resumable-media-python/commit/a84069127cd48f68e3a56b3df16c63ff494637f3))
+
+## [1.3.0](https://www.github.com/googleapis/google-resumable-media-python/compare/v1.2.0...v1.3.0) (2021-05-18)
+
+
+### Features
+
+* allow RetryStrategy to be configured with a custom initial wait and multiplier ([#216](https://www.github.com/googleapis/google-resumable-media-python/issues/216)) ([579a54b](https://www.github.com/googleapis/google-resumable-media-python/commit/579a54b56dd7045da7af0dcacacfa5833c1cfa87))
+
+
+### Documentation
+
+* address terminology ([#201](https://www.github.com/googleapis/google-resumable-media-python/issues/201)) ([a88cfb9](https://www.github.com/googleapis/google-resumable-media-python/commit/a88cfb9637015839307ea4e967eef6f232c007a5))
+
+## [1.2.0](https://www.github.com/googleapis/google-resumable-media-python/compare/v1.1.0...v1.2.0) (2020-12-14)
+
+
+### Features
+
+* add support for Python 3.9, drop support for Python 3.5 ([#191](https://www.github.com/googleapis/google-resumable-media-python/issues/191)) ([76839fb](https://www.github.com/googleapis/google-resumable-media-python/commit/76839fb9bf6fd57ec6bea7b82aeaa1b3fe6f4464)), closes [#189](https://www.github.com/googleapis/google-resumable-media-python/issues/189)
+* add retries for 'requests.ConnectionError' ([#186](https://www.github.com/googleapis/google-resumable-media-python/issues/186)) ([0d76eac](https://www.github.com/googleapis/google-resumable-media-python/commit/0d76eac29758d119e292fb27ab8000432944a938))
+
+## [1.1.0](https://www.github.com/googleapis/google-resumable-media-python/compare/v1.0.0...v1.1.0) (2020-10-05)
+
+
+### Features
+
+* add _async_resumable_media experimental support ([#179](https://www.github.com/googleapis/google-resumable-media-python/issues/179)) ([03c11ba](https://www.github.com/googleapis/google-resumable-media-python/commit/03c11bae0c43830d539f1e0adcc837a6c88f4e2e)), closes [#160](https://www.github.com/googleapis/google-resumable-media-python/issues/160) [#153](https://www.github.com/googleapis/google-resumable-media-python/issues/153) [#176](https://www.github.com/googleapis/google-resumable-media-python/issues/176) [#178](https://www.github.com/googleapis/google-resumable-media-python/issues/178)
+
+
+### Bug Fixes
+
+* allow space in checksum header ([#170](https://www.github.com/googleapis/google-resumable-media-python/issues/170)) ([224fc98](https://www.github.com/googleapis/google-resumable-media-python/commit/224fc9858b903396e0f94801757814e47cff45e7)), closes [#169](https://www.github.com/googleapis/google-resumable-media-python/issues/169)
+* **lint:** blacken 5 files ([#171](https://www.github.com/googleapis/google-resumable-media-python/issues/171)) ([cdea3ee](https://www.github.com/googleapis/google-resumable-media-python/commit/cdea3eec76c7586a66b1641bca906f630d915c0e))
+
+## [1.0.0](https://www.github.com/googleapis/google-resumable-media-python/compare/v0.7.1...v1.0.0) (2020-08-24)
+
+
+### Features
+
+* bump 'google-crc32c >= 1.0' ([#162](https://www.github.com/googleapis/google-resumable-media-python/issues/162)) ([eaf9faa](https://www.github.com/googleapis/google-resumable-media-python/commit/eaf9faa80dc51bd719161557584e151b30c7e082))
+
+## [0.7.1](https://www.github.com/googleapis/google-resumable-media-python/compare/v0.7.0...v0.7.1) (2020-08-06)
+
+
+### Dependencies
+* pin 'google-crc32c < 0.2dev' ([#160](https://www.github.com/googleapis/google-resumable-media-python/issues/160)) ([52a322d](https://www.github.com/googleapis/google-resumable-media-python/commit/52a322d478e074a646e20d92ca9b2457c6e03941))
+
+### Documentation
+
+* update docs build (via synth) ([#155](https://www.github.com/googleapis/google-resumable-media-python/issues/155)) ([1c33de4](https://www.github.com/googleapis/google-resumable-media-python/commit/1c33de475585e27bba2bcb7ea5dbead9e0214660))
+* use googleapis.dev docs link ([#149](https://www.github.com/googleapis/google-resumable-media-python/issues/149)) ([90bd0c1](https://www.github.com/googleapis/google-resumable-media-python/commit/90bd0c1a6a88b53c2049cd75cf73129fcecde5de))
+
+## [0.7.0](https://www.github.com/googleapis/google-resumable-media-python/compare/v0.6.0...v0.7.0) (2020-07-23)
+
+
+### Features
+
+* add configurable checksum support for uploads ([#139](https://www.github.com/googleapis/google-resumable-media-python/issues/139)) ([68264f8](https://www.github.com/googleapis/google-resumable-media-python/commit/68264f811473a5aa06102912ea8d454b6bb59307))
+
+
+### Bug Fixes
+
+* accept `201 Created` as valid upload response ([#141](https://www.github.com/googleapis/google-resumable-media-python/issues/141)) ([00d280e](https://www.github.com/googleapis/google-resumable-media-python/commit/00d280e116d69f7f8d8a4b970bb1176e338f9ce0)), closes [#125](https://www.github.com/googleapis/google-resumable-media-python/issues/125) [#124](https://www.github.com/googleapis/google-resumable-media-python/issues/124)
+
+## [0.6.0](https://www.github.com/googleapis/google-resumable-media-python/compare/v0.5.1...v0.6.0) (2020-07-07)
+
+
+### Features
+
+* add customizable timeouts to upload/download methods ([#116](https://www.github.com/googleapis/google-resumable-media-python/issues/116)) ([5310921](https://www.github.com/googleapis/google-resumable-media-python/commit/5310921dacf5232ad58a5d6e324f3df6b320eca7))
+* add configurable crc32c checksumming for downloads ([#135](https://www.github.com/googleapis/google-resumable-media-python/issues/135)) ([db31bf5](https://www.github.com/googleapis/google-resumable-media-python/commit/db31bf56560109f19b49cf38c494ecacfa74b0b2))
+* add templates for python samples projects ([#506](https://www.github.com/googleapis/google-resumable-media-python/issues/506)) ([#132](https://www.github.com/googleapis/google-resumable-media-python/issues/132)) ([8e60cc4](https://www.github.com/googleapis/google-resumable-media-python/commit/8e60cc45d4fefe63d7353e978d15d6e238b7a1a1))
+
+
+### Documentation
+
+* update client_documentation link ([#136](https://www.github.com/googleapis/google-resumable-media-python/issues/136)) ([063b4f9](https://www.github.com/googleapis/google-resumable-media-python/commit/063b4f9f3ea3dff850d9ae46b2abf25d08312320))
+
+## [0.5.1](https://www.github.com/googleapis/google-resumable-media-python/compare/v0.5.0...v0.5.1) (2020-05-26)
+
+
+### Bug Fixes
+
+* fix failing unit tests by dropping Python 3.4, add Python 3.8 ([#118](https://www.github.com/googleapis/google-resumable-media-python/issues/118)) ([1edb974](https://www.github.com/googleapis/google-resumable-media-python/commit/1edb974175d16c9f542fe84dd6bbfa2d70115d48))
+* fix upload_from_file size greater than multipart ([#129](https://www.github.com/googleapis/google-resumable-media-python/issues/129)) ([07dd9c2](https://www.github.com/googleapis/google-resumable-media-python/commit/07dd9c26a7eff9b2b43d32636faf9a5aa151fed5))
+* Generated file update for docs and testing templates. ([#127](https://www.github.com/googleapis/google-resumable-media-python/issues/127)) ([bc7a5a9](https://www.github.com/googleapis/google-resumable-media-python/commit/bc7a5a9b66e16d08778ace96845bb429b94ddbce))
+
+## 0.5.0
+
+10-28-2019 09:16 PDT
+
+
+### New Features
+- Add raw download classes. ([#109](https://github.com/googleapis/google-resumable-media-python/pull/109))
+
+### Documentation
+- Update Sphinx inventory URL for requests library. ([#108](https://github.com/googleapis/google-resumable-media-python/pull/108))
+
+### Internal / Testing Changes
+- Initial synth. ([#105](https://github.com/googleapis/google-resumable-media-python/pull/105))
+- Remove CircleCI. ([#102](https://github.com/googleapis/google-resumable-media-python/pull/102))
+
+## 0.4.1
+
+09-16-2019 17:59 PDT
+
+
+### Implementation Changes
+- Revert "Always use raw response data. ([#87](https://github.com/googleapis/google-resumable-media-python/pull/87))" ([#103](https://github.com/googleapis/google-resumable-media-python/pull/103))
+
+### Internal / Testing Changes
+- Add black. ([#94](https://github.com/googleapis/google-resumable-media-python/pull/94))
+
+## 0.4.0
+
+09-05-2019 11:59 PDT
+
+### Backward-Compatibility Note
+
+The change to use raw response data (PR
+[#87](https://github.com/googleapis/google-resumable-media-python/pull/87))
+might break the hypothetical usecase of downloading a blob marked with
+`Content-Encoding: gzip` and expecting to get the expanded data.
+
+### Implementation Changes
+- Require 200 response for initial resumable upload request. ([#95](https://github.com/googleapis/google-resumable-media-python/pull/95))
+- Use `response` as variable for object returned from `http_request`. ([#98](https://github.com/googleapis/google-resumable-media-python/pull/98))
+- Further DRY request dependency pins. ([#96](https://github.com/googleapis/google-resumable-media-python/pull/96))
+- Finish download on seeing 416 response with zero byte range. ([#86](https://github.com/googleapis/google-resumable-media-python/pull/86))
+- Always use raw response data. ([#87](https://github.com/googleapis/google-resumable-media-python/pull/87))
+
+### Dependencies
+- Drop runtime dependency check on `requests`. ([#97](https://github.com/googleapis/google-resumable-media-python/pull/97))
+
+### Documentation
+- Update docs after release ([#93](https://github.com/googleapis/google-resumable-media-python/pull/93))
+
+## 0.3.3
+
+08-23-2019 14:15 PDT
+
+### Implementation Changes
+- Add a default timeout for the http_request method ([#88](https://github.com/googleapis/google-resumable-media-python/pull/88))
+- DRY 'requests' pin; don't shadow exception. ([#83](https://github.com/googleapis/google-resumable-media-python/pull/83))
+- Drop a hardcoded value in an error message. ([#48](https://github.com/googleapis/google-resumable-media-python/pull/48))
+
+### Documentation
+- Reconstruct 'CHANGELOG.md' from pre-releasetool era releases. ([#66](https://github.com/googleapis/google-resumable-media-python/pull/66))
+
+### Internal / Testing Changes
+- Use Kokoro for CI ([#90](https://github.com/googleapis/google-resumable-media-python/pull/90))
+- Renovate: preserve semver ranges. ([#82](https://github.com/googleapis/google-resumable-media-python/pull/82))
+- Add renovate.json ([#79](https://github.com/googleapis/google-resumable-media-python/pull/79))
+- Fix systest bitrot. ([#77](https://github.com/googleapis/google-resumable-media-python/pull/77))
+- Fix docs build redux. ([#75](https://github.com/googleapis/google-resumable-media-python/pull/75))
+- Update to new nox ([#57](https://github.com/googleapis/google-resumable-media-python/pull/57))
+
+## 0.3.2
+
+2018-12-17 17:31 PST
+
+### Implementation Changes
+- Using `str` instead of `repr` for multipart boundary.
+
+### Dependencies
+- Making `requests` a strict dependency for the `requests` subpackage.
+
+### Documentation
+- Announce deprecation of Python 2.7 ([#51](https://github.com/googleapis/google-resumable-media-python/pull/51))
+- Fix broken redirect after repository move
+- Updating generated static content in docs.
+
+### Internal / Testing Changes
+- Modify file not found test to look for the correct error message
+- Harden tests so they can run with debug logging statements
+- Add Appveyor support. ([#40](https://github.com/googleapis/google-resumable-media-python/pull/40))
+- Mark the version in `main` as `.dev1`.
+
+
+## 0.3.1
+
+2017-10-20
+
+### Implementation Changes
+
+- Add requests/urllib3 work-around for intercepting gzipped bytes. ([#36](https://github.com/googleapis/google-resumable-media-python/pull/36))
+
+### Internal / Testing Changes
+- Re-factor `system/requests/test_download.py`. ([#35](https://github.com/googleapis/google-resumable-media-python/pull/35))
+
+
+## 0.3.0
+
+2017-10-13
+
+### Implementation Changes
+
+- Add checksum validation for non-chunked non-composite downloads. ([#32](https://github.com/googleapis/google-resumable-media-python/pull/32))
+
+### Dependencies
+
+- Add `requests` extra to `setup.py`. ([#30](https://github.com/googleapis/google-resumable-media-python/pull/30))
+
+### Documentation
+
+- Update generated docs, due to updated `objects.inf` from reequests.
+
+
+## 0.2.3
+
+2017-08-07
+
+### Implementation Changes
+
+- Allow empty files to be uploaded. ([#25](https://github.com/googleapis/google-resumable-media-python/pull/25))
+
+
+## 0.2.2
+
+2017-08-01
+
+### Implementation Changes
+
+- Swap the order of `_write_to_stream()` / `_process_response()` in `requests` download. ([#24](https://github.com/googleapis/google-resumable-media-python/pull/24))
+- Use requests `iter_content()` to avoid storing response body in RAM. ([#21](https://github.com/googleapis/google-resumable-media-python/pull/21))
+- Add optional stream argument to DownloadBase. ([#20](https://github.com/googleapis/google-resumable-media-python/pull/20))
+
+
+## 0.2.1
+
+2017-07-21
+
+### Implementation Changes
+
+- Drop usage of `size` to measure (resumable) bytes uploaded. ([#18](https://github.com/googleapis/google-resumable-media-python/pull/18))
+- Use explicit u prefix on unicode strings. ([#16](https://github.com/googleapis/google-resumable-media-python/pull/16))
+
+### Internal / Testing Changes
+
+- Update `author_email1` to official mailing list.
+
+## 0.2.0
+
+2017-07-18
+
+### Implementation Changes
+
+- Ensure passing unicode to `json.loads()` rather than `bytes`. ([#13](https://github.com/googleapis/google-resumable-media-python/pull/13))
+- Add `MANIFEST.in` to repository. ([#9](https://github.com/googleapis/google-resumable-media-python/pull/9))
+- Move contents of exceptions module into common.
+
+### Documentation
+
+- Update docs after latest version of Sphinx. ([#11](https://github.com/googleapis/google-resumable-media-python/pull/11))
+- Update `custom_html_writer` after Sphinx update removed a class. ([#10](https://github.com/googleapis/google-resumable-media-python/pull/10))
+
+### Internal / Testing Changes
+
+- Use nox `session.skip` (instead of ValueError) for system tests. ([#14](https://github.com/googleapis/google-resumable-media-python/pull/14))
+
+
+## 0.1.1
+
+2017-05-05
+
+
+### Implementation Changes
+
+- Add `common.RetryStrategy` class; us it in `wait_and_retry`.
+- Rename `constants` module -> `common`.
+
+
+## 0.1.0
+
+2017-05-03
+
+### Implementation Changes
+
+- Pass `total_bytes` in `requests.ResumableUpload.initiate`.
+
+
+## 0.0.5
+
+2017-05-02
+
+### New Features
+
+- Add support for resumable uploads of unknown size. ([#6](https://github.com/googleapis/google-resumable-media-python/pull/6))
+
+
+## 0.0.4
+
+2017-04-28
+
+### Implementation Changes
+
+- Refactor upload / download support into public, transport-agnostic classes and private, `requests`-specific implementations.
+
+
+## 0.0.3
+
+2017-04-24
+
+### New Features
+
+- Add automatic retries for 429, 500, 502, 503 and 504 error responses. ([#4](https://github.com/googleapis/google-resumable-media-python/pull/4))
+
+
+## 0.0.2
+
+2017-04-24
+
+### New Features
+
+- Add optional `headers` to upload / download classes.
+
+### Documentation
+
+- Automate documentation builds via CircleCI.
+
+
+## 0.0.1
+
+2017-04-21
+
+- Initial public release.
diff --git a/packages/google-resumable-media/CONTRIBUTING.md b/packages/google-resumable-media/CONTRIBUTING.md
new file mode 100644
index 000000000000..6272489dae31
--- /dev/null
+++ b/packages/google-resumable-media/CONTRIBUTING.md
@@ -0,0 +1,28 @@
+# How to Contribute
+
+We'd love to accept your patches and contributions to this project. There are
+just a few small guidelines you need to follow.
+
+## Contributor License Agreement
+
+Contributions to this project must be accompanied by a Contributor License
+Agreement. You (or your employer) retain the copyright to your contribution;
+this simply gives us permission to use and redistribute your contributions as
+part of the project. Head over to to see
+your current agreements on file or to sign a new one.
+
+You generally only need to submit a CLA once, so if you've already submitted one
+(even if it was for a different project), you probably don't need to do it
+again.
+
+## Code Reviews
+
+All submissions, including submissions by project members, require review. We
+use GitHub pull requests for this purpose. Consult
+[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
+information on using pull requests.
+
+## Community Guidelines
+
+This project follows [Google's Open Source Community
+Guidelines](https://opensource.google/conduct/).
diff --git a/packages/google-resumable-media/LICENSE b/packages/google-resumable-media/LICENSE
new file mode 100644
index 000000000000..d64569567334
--- /dev/null
+++ b/packages/google-resumable-media/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/packages/google-resumable-media/MANIFEST.in b/packages/google-resumable-media/MANIFEST.in
new file mode 100644
index 000000000000..01511ce8adad
--- /dev/null
+++ b/packages/google-resumable-media/MANIFEST.in
@@ -0,0 +1,4 @@
+include README.rst LICENSE
+recursive-include google
+recursive-include tests *
+global-exclude *.pyc __pycache__
diff --git a/packages/google-resumable-media/README.rst b/packages/google-resumable-media/README.rst
new file mode 100644
index 000000000000..99e9fbf3e377
--- /dev/null
+++ b/packages/google-resumable-media/README.rst
@@ -0,0 +1,37 @@
+``google-resumable-media``
+==========================
+
+
+Utilities for Google Media Downloads and Resumable Uploads
+
+
+See the `docs`_ for examples and usage.
+
+.. _docs: https://googleapis.dev/python/google-resumable-media/latest/index.html
+
+Experimental `asyncio` Support
+------------------------------
+While still in development and subject to change, this library has `asyncio`
+support at `google._async_resumable_media`.
+
+Supported Python Versions
+-------------------------
+Python >= 3.7
+
+Unsupported Python Versions
+---------------------------
+
+Python == 2.7, Python == 3.5, Python == 3.6.
+
+The last version of this library compatible with Python 2.7 and 3.5 is
+`google-resumable-media==1.3.3`.
+
+The last version of this library compatible with Python 3.6 is
+`google-resumable-media==2.3.3`.
+
+License
+-------
+
+Apache 2.0 - See `the LICENSE`_ for more information.
+
+.. _the LICENSE: https://github.com/googleapis/google-resumable-media-python/blob/main/LICENSE
diff --git a/packages/google-resumable-media/docs/CHANGELOG.md b/packages/google-resumable-media/docs/CHANGELOG.md
new file mode 120000
index 000000000000..04c99a55caae
--- /dev/null
+++ b/packages/google-resumable-media/docs/CHANGELOG.md
@@ -0,0 +1 @@
+../CHANGELOG.md
\ No newline at end of file
diff --git a/packages/google-resumable-media/docs/README.rst b/packages/google-resumable-media/docs/README.rst
new file mode 120000
index 000000000000..89a0106941ff
--- /dev/null
+++ b/packages/google-resumable-media/docs/README.rst
@@ -0,0 +1 @@
+../README.rst
\ No newline at end of file
diff --git a/packages/google-resumable-media/docs/_static/custom.css b/packages/google-resumable-media/docs/_static/custom.css
new file mode 100644
index 000000000000..b0a295464b23
--- /dev/null
+++ b/packages/google-resumable-media/docs/_static/custom.css
@@ -0,0 +1,20 @@
+div#python2-eol {
+ border-color: red;
+ border-width: medium;
+}
+
+/* Ensure minimum width for 'Parameters' / 'Returns' column */
+dl.field-list > dt {
+ min-width: 100px
+}
+
+/* Insert space between methods for readability */
+dl.method {
+ padding-top: 10px;
+ padding-bottom: 10px
+}
+
+/* Insert empty space between classes */
+dl.class {
+ padding-bottom: 50px
+}
diff --git a/packages/google-resumable-media/docs/_templates/layout.html b/packages/google-resumable-media/docs/_templates/layout.html
new file mode 100644
index 000000000000..6316a537f72b
--- /dev/null
+++ b/packages/google-resumable-media/docs/_templates/layout.html
@@ -0,0 +1,50 @@
+
+{% extends "!layout.html" %}
+{%- block content %}
+{%- if theme_fixed_sidebar|lower == 'true' %}
+
+ As of January 1, 2020 this library no longer supports Python 2 on the latest released version.
+ Library versions released prior to that date will continue to be available. For more information please
+ visit Python 2 support on Google Cloud.
+
+{%- else %}
+{{ super() }}
+{%- endif %}
+{%- endblock %}
diff --git a/packages/google-resumable-media/docs/conf.py b/packages/google-resumable-media/docs/conf.py
new file mode 100644
index 000000000000..c16787e3528c
--- /dev/null
+++ b/packages/google-resumable-media/docs/conf.py
@@ -0,0 +1,369 @@
+# -*- coding: utf-8 -*-
+#
+# google-resumable-media documentation build configuration file
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+import shlex
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath(".."))
+
+# For plugins that can not read conf.py.
+# See also: https://github.com/docascode/sphinx-docfx-yaml/issues/85
+sys.path.insert(0, os.path.abspath("."))
+
+__version__ = ""
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+needs_sphinx = "1.5.5"
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+ "sphinx.ext.autodoc",
+ "sphinx.ext.autosummary",
+ "sphinx.ext.intersphinx",
+ "sphinx.ext.coverage",
+ "sphinx.ext.napoleon",
+ "sphinx.ext.todo",
+ "sphinx.ext.viewcode",
+ "recommonmark",
+ "sphinx.ext.doctest"
+]
+
+# autodoc/autosummary flags
+autoclass_content = "both"
+autodoc_default_options = {"members": True}
+autosummary_generate = True
+
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
+source_suffix = [".rst", ".md"]
+
+# The encoding of source files.
+# source_encoding = 'utf-8-sig'
+
+# The main toctree document.
+root_doc = "index"
+
+# General information about the project.
+project = u"google-resumable-media"
+copyright = u"2019, Google"
+author = u"Google APIs"
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The full version, including alpha/beta/rc tags.
+release = __version__
+# The short X.Y version.
+version = ".".join(release.split(".")[0:2])
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+# today = ''
+# Else, today_fmt is used as the format for a strftime call.
+# today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = [
+ "_build",
+ "samples/AUTHORING_GUIDE.md",
+ "samples/CONTRIBUTING.md",
+ "samples/snippets/README.rst",
+]
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+# default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+# add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+# add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+# show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = "sphinx"
+
+# A list of ignored prefixes for module index sorting.
+# modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+# keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = "alabaster"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {
+ "description": "Google Cloud Client Libraries for google-resumable-media",
+ "github_user": "googleapis",
+ "github_repo": "google-resumable-media-python",
+ "github_banner": True,
+ "font_family": "'Roboto', Georgia, sans",
+ "head_font_family": "'Roboto', Georgia, serif",
+ "code_font_family": "'Roboto Mono', 'Consolas', monospace",
+}
+
+# Add any paths that contain custom themes here, relative to this directory.
+# html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# " v documentation".
+# html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+# html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+# html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+# html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+# html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+# html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+# html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+# html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+# html_additional_pages = {}
+
+# If false, no module index is generated.
+# html_domain_indices = True
+
+# If false, no index is generated.
+# html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+# html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+# html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+# html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+# html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+# html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+# html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
+# html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# Now only 'ja' uses this config value
+# html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+# html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = "google-resumable-media-doc"
+
+# -- Options for warnings ------------------------------------------------------
+
+
+suppress_warnings = [
+ # Temporarily suppress this to avoid "more than one target found for
+ # cross-reference" warning, which are intractable for us to avoid while in
+ # a mono-repo.
+ # See https://github.com/sphinx-doc/sphinx/blob
+ # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843
+ "ref.python"
+]
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+ # The paper size ('letterpaper' or 'a4paper').
+ #'papersize': 'letterpaper',
+ # The font size ('10pt', '11pt' or '12pt').
+ #'pointsize': '10pt',
+ # Additional stuff for the LaTeX preamble.
+ #'preamble': '',
+ # Latex figure (float) alignment
+ #'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+# author, documentclass [howto, manual, or own class]).
+latex_documents = [
+ (
+ root_doc,
+ "google-resumable-media.tex",
+ u"google-resumable-media Documentation",
+ author,
+ "manual",
+ )
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+# latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+# latex_use_parts = False
+
+# If true, show page references after internal links.
+# latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+# latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+# latex_appendices = []
+
+# If false, no module index is generated.
+# latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ (
+ root_doc,
+ "google-resumable-media",
+ u"google-resumable-media Documentation",
+ [author],
+ 1,
+ )
+]
+
+# If true, show URL addresses after external links.
+# man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ (
+ root_doc,
+ "google-resumable-media",
+ u"google-resumable-media Documentation",
+ author,
+ "google-resumable-media",
+ "google-resumable-media Library",
+ "APIs",
+ )
+]
+
+# Documents to append as an appendix to all manuals.
+# texinfo_appendices = []
+
+# If false, no module index is generated.
+# texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+# texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+# texinfo_no_detailmenu = False
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {
+ "python": ("http://python.readthedocs.org/en/latest/", None),
+ "google-auth": ("https://google-auth.readthedocs.io/en/stable", None),
+ "google.api_core": (
+ "https://googleapis.dev/python/google-api-core/latest/",
+ None,
+ ),
+ "grpc": ("https://grpc.io/grpc/python/", None),
+
+}
+
+
+# Napoleon settings
+napoleon_google_docstring = True
+napoleon_numpy_docstring = True
+napoleon_include_private_with_doc = False
+napoleon_include_special_with_doc = True
+napoleon_use_admonition_for_examples = False
+napoleon_use_admonition_for_notes = False
+napoleon_use_admonition_for_references = False
+napoleon_use_ivar = False
+napoleon_use_param = True
+napoleon_use_rtype = True
diff --git a/packages/google-resumable-media/docs/index.rst b/packages/google-resumable-media/docs/index.rst
new file mode 100644
index 000000000000..cbe7eae128df
--- /dev/null
+++ b/packages/google-resumable-media/docs/index.rst
@@ -0,0 +1,22 @@
+.. include:: README.rst
+
+
+
+API Reference
+----------------
+.. toctree::
+ :maxdepth: 2
+
+ resumable_media/common
+ resumable_media/requests
+
+
+Changelog
+---------
+
+For a list of all ``google-resumable-media`` releases.
+
+.. toctree::
+ :maxdepth: 2
+
+ CHANGELOG
\ No newline at end of file
diff --git a/packages/google-resumable-media/docs/resumable_media/common.rst b/packages/google-resumable-media/docs/resumable_media/common.rst
new file mode 100644
index 000000000000..23a98e1e356f
--- /dev/null
+++ b/packages/google-resumable-media/docs/resumable_media/common.rst
@@ -0,0 +1,6 @@
+Common Utilities
+================
+
+.. automodule:: google.resumable_media.common
+ :members:
+ :inherited-members:
\ No newline at end of file
diff --git a/packages/google-resumable-media/docs/resumable_media/requests.rst b/packages/google-resumable-media/docs/resumable_media/requests.rst
new file mode 100644
index 000000000000..ea454b2d5251
--- /dev/null
+++ b/packages/google-resumable-media/docs/resumable_media/requests.rst
@@ -0,0 +1,6 @@
+Requests Utilities
+===================
+
+.. automodule:: google.resumable_media.requests
+ :members:
+ :inherited-members:
\ No newline at end of file
diff --git a/packages/google-resumable-media/google/_async_resumable_media/__init__.py b/packages/google-resumable-media/google/_async_resumable_media/__init__.py
new file mode 100644
index 000000000000..eaade3e8792f
--- /dev/null
+++ b/packages/google-resumable-media/google/_async_resumable_media/__init__.py
@@ -0,0 +1,60 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for Google Media Downloads and Resumable Uploads.
+
+This package has some general purposes modules, e.g.
+:mod:`~google.resumable_media.common`, but the majority of the
+public interface will be contained in subpackages.
+
+===========
+Subpackages
+===========
+
+Each subpackage is tailored to a specific transport library:
+
+* the :mod:`~google.resumable_media.requests` subpackage uses the ``requests``
+ transport library.
+
+.. _requests: http://docs.python-requests.org/
+
+==========
+Installing
+==========
+
+To install with `pip`_:
+
+.. code-block:: console
+
+ $ pip install --upgrade google-resumable-media
+
+.. _pip: https://pip.pypa.io/
+"""
+
+from google.resumable_media.common import DataCorruption
+from google.resumable_media.common import InvalidResponse
+from google.resumable_media.common import PERMANENT_REDIRECT
+from google.resumable_media.common import RetryStrategy
+from google.resumable_media.common import TOO_MANY_REQUESTS
+from google.resumable_media.common import UPLOAD_CHUNK_SIZE
+
+
+__all__ = [
+ "DataCorruption",
+ "InvalidResponse",
+ "PERMANENT_REDIRECT",
+ "RetryStrategy",
+ "TOO_MANY_REQUESTS",
+ "UPLOAD_CHUNK_SIZE",
+]
diff --git a/packages/google-resumable-media/google/_async_resumable_media/_download.py b/packages/google-resumable-media/google/_async_resumable_media/_download.py
new file mode 100644
index 000000000000..1966f339c81c
--- /dev/null
+++ b/packages/google-resumable-media/google/_async_resumable_media/_download.py
@@ -0,0 +1,550 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Virtual bases classes for downloading media from Google APIs."""
+
+import http.client
+import re
+
+from google._async_resumable_media import _helpers
+from google.resumable_media import common
+
+
+_CONTENT_RANGE_RE = re.compile(
+ r"bytes (?P\d+)-(?P\d+)/(?P\d+)",
+ flags=re.IGNORECASE,
+)
+_ACCEPTABLE_STATUS_CODES = (http.client.OK, http.client.PARTIAL_CONTENT)
+_GET = "GET"
+_ZERO_CONTENT_RANGE_HEADER = "bytes */0"
+
+
+class DownloadBase(object):
+ """Base class for download helpers.
+
+ Defines core shared behavior across different download types.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ the downloaded resource can be written to.
+ start (int): The first byte in a range to be downloaded.
+ end (int): The last byte in a range to be downloaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ """
+
+ def __init__(self, media_url, stream=None, start=None, end=None, headers=None):
+ self.media_url = media_url
+ self._stream = stream
+ self.start = start
+ self.end = end
+ if headers is None:
+ headers = {}
+ self._headers = headers
+ self._finished = False
+ self._retry_strategy = common.RetryStrategy()
+
+ @property
+ def finished(self):
+ """bool: Flag indicating if the download has completed."""
+ return self._finished
+
+ @staticmethod
+ def _get_status_code(response):
+ """Access the status code from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ @staticmethod
+ def _get_headers(response):
+ """Access the headers from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ @staticmethod
+ def _get_body(response):
+ """Access the response body from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class Download(DownloadBase):
+ """Helper to manage downloading a resource from a Google API.
+
+ "Slices" of the resource can be retrieved by specifying a range
+ with ``start`` and / or ``end``. However, in typical usage, neither
+ ``start`` nor ``end`` is expected to be provided.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ the downloaded resource can be written to.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, but ``end`` is provided, will download from the
+ beginning to ``end`` of the media.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, but ``start`` is provided, will download from the
+ ``start`` to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The response headers must contain
+ a checksum of the requested type. If the headers lack an
+ appropriate checksum (for instance in the case of transcoded or
+ ranged downloads where the remote service does not know the
+ correct checksum) an INFO-level log will be emitted. Supported
+ values are "md5", "crc32c" and None.
+ """
+
+ def __init__(
+ self, media_url, stream=None, start=None, end=None, headers=None, checksum="md5"
+ ):
+ super(Download, self).__init__(
+ media_url, stream=stream, start=start, end=end, headers=headers
+ )
+ self.checksum = checksum
+
+ def _prepare_request(self):
+ """Prepare the contents of an HTTP request.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Returns:
+ Tuple[str, str, NoneType, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always GET)
+ * the URL for the request
+ * the body of the request (always :data:`None`)
+ * headers for the request
+
+ Raises:
+ ValueError: If the current :class:`Download` has already
+ finished.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("A download can only be used once.")
+
+ add_bytes_range(self.start, self.end, self._headers)
+ return _GET, self.media_url, None, self._headers
+
+ def _process_response(self, response):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ # Tombstone the current Download so it cannot be used again.
+ self._finished = True
+ _helpers.require_status_code(
+ response, _ACCEPTABLE_STATUS_CODES, self._get_status_code
+ )
+
+ def consume(self, transport, timeout=None):
+ """Consume the resource to be downloaded.
+
+ If a ``stream`` is attached to this download, then the downloaded
+ resource will be written to the stream.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class ChunkedDownload(DownloadBase):
+ """Download a resource in chunks from a Google API.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each
+ request.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ will be used to concatenate chunks of the resource as they are
+ downloaded.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, defaults to ``0``.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, will download to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with each request, e.g. headers for data encryption
+ key headers.
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each request.
+
+ Raises:
+ ValueError: If ``start`` is negative.
+ """
+
+ def __init__(self, media_url, chunk_size, stream, start=0, end=None, headers=None):
+ if start < 0:
+ raise ValueError(
+ "On a chunked download the starting value cannot be negative."
+ )
+ super(ChunkedDownload, self).__init__(
+ media_url, stream=stream, start=start, end=end, headers=headers
+ )
+ self.chunk_size = chunk_size
+ self._bytes_downloaded = 0
+ self._total_bytes = None
+ self._invalid = False
+
+ @property
+ def bytes_downloaded(self):
+ """int: Number of bytes that have been downloaded."""
+ return self._bytes_downloaded
+
+ @property
+ def total_bytes(self):
+ """Optional[int]: The total number of bytes to be downloaded."""
+ return self._total_bytes
+
+ @property
+ def invalid(self):
+ """bool: Indicates if the download is in an invalid state.
+
+ This will occur if a call to :meth:`consume_next_chunk` fails.
+ """
+ return self._invalid
+
+ def _get_byte_range(self):
+ """Determines the byte range for the next request.
+
+ Returns:
+ Tuple[int, int]: The pair of begin and end byte for the next
+ chunked request.
+ """
+ curr_start = self.start + self.bytes_downloaded
+ curr_end = curr_start + self.chunk_size - 1
+ # Make sure ``curr_end`` does not exceed ``end``.
+ if self.end is not None:
+ curr_end = min(curr_end, self.end)
+ # Make sure ``curr_end`` does not exceed ``total_bytes - 1``.
+ if self.total_bytes is not None:
+ curr_end = min(curr_end, self.total_bytes - 1)
+ return curr_start, curr_end
+
+ def _prepare_request(self):
+ """Prepare the contents of an HTTP request.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ .. note:
+
+ This method will be used multiple times, so ``headers`` will
+ be mutated in between requests. However, we don't make a copy
+ since the same keys are being updated.
+
+ Returns:
+ Tuple[str, str, NoneType, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always GET)
+ * the URL for the request
+ * the body of the request (always :data:`None`)
+ * headers for the request
+
+ Raises:
+ ValueError: If the current download has finished.
+ ValueError: If the current download is invalid.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("Download has finished.")
+ if self.invalid:
+ raise ValueError("Download is invalid and cannot be re-used.")
+
+ curr_start, curr_end = self._get_byte_range()
+ add_bytes_range(curr_start, curr_end, self._headers)
+ return _GET, self.media_url, None, self._headers
+
+ def _make_invalid(self):
+ """Simple setter for ``invalid``.
+
+ This is intended to be passed along as a callback to helpers that
+ raise an exception so they can mark this instance as invalid before
+ raising.
+ """
+ self._invalid = True
+
+ async def _process_response(self, response):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O. This is based on the `sans-I/O`_ philosophy.
+
+ For the time being, this **does require** some form of I/O to write
+ a chunk to ``stream``. However, this will (almost) certainly not be
+ network I/O.
+
+ Updates the current state after consuming a chunk. First,
+ increments ``bytes_downloaded`` by the number of bytes in the
+ ``content-length`` header.
+
+ If ``total_bytes`` is already set, this assumes (but does not check)
+ that we already have the correct value and doesn't bother to check
+ that it agrees with the headers.
+
+ We expect the **total** length to be in the ``content-range`` header,
+ but this header is only present on requests which sent the ``range``
+ header. This response header should be of the form
+ ``bytes {start}-{end}/{total}`` and ``{end} - {start} + 1``
+ should be the same as the ``Content-Length``.
+
+ Args:
+ response (object): The HTTP response object (need headers).
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the number
+ of bytes in the body doesn't match the content length header.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ # Verify the response before updating the current instance.
+ if _check_for_zero_content_range(
+ response, self._get_status_code, self._get_headers
+ ):
+ self._finished = True
+ return
+
+ _helpers.require_status_code(
+ response,
+ _ACCEPTABLE_STATUS_CODES,
+ self._get_status_code,
+ callback=self._make_invalid,
+ )
+ headers = self._get_headers(response)
+ response_body = await self._get_body(response)
+
+ start_byte, end_byte, total_bytes = get_range_info(
+ response, self._get_headers, callback=self._make_invalid
+ )
+
+ transfer_encoding = headers.get("transfer-encoding")
+
+ if transfer_encoding is None:
+ content_length = _helpers.header_required(
+ response,
+ "content-length",
+ self._get_headers,
+ callback=self._make_invalid,
+ )
+ num_bytes = int(content_length)
+
+ if len(response_body) != num_bytes:
+ self._make_invalid()
+ raise common.InvalidResponse(
+ response,
+ "Response is different size than content-length",
+ "Expected",
+ num_bytes,
+ "Received",
+ len(response_body),
+ )
+ else:
+ # 'content-length' header not allowed with chunked encoding.
+ num_bytes = end_byte - start_byte + 1
+
+ # First update ``bytes_downloaded``.
+ self._bytes_downloaded += num_bytes
+ # If the end byte is past ``end`` or ``total_bytes - 1`` we are done.
+ if self.end is not None and end_byte >= self.end:
+ self._finished = True
+ elif end_byte >= total_bytes - 1:
+ self._finished = True
+ # NOTE: We only use ``total_bytes`` if not already known.
+ if self.total_bytes is None:
+ self._total_bytes = total_bytes
+ # Write the response body to the stream.
+ self._stream.write(response_body)
+
+ def consume_next_chunk(self, transport, timeout=None):
+ """Consume the next chunk of the resource to be downloaded.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+def add_bytes_range(start, end, headers):
+ """Add a bytes range to a header dictionary.
+
+ Some possible inputs and the corresponding bytes ranges::
+
+ >>> headers = {}
+ >>> add_bytes_range(None, None, headers)
+ >>> headers
+ {}
+ >>> add_bytes_range(500, 999, headers)
+ >>> headers['range']
+ 'bytes=500-999'
+ >>> add_bytes_range(None, 499, headers)
+ >>> headers['range']
+ 'bytes=0-499'
+ >>> add_bytes_range(-500, None, headers)
+ >>> headers['range']
+ 'bytes=-500'
+ >>> add_bytes_range(9500, None, headers)
+ >>> headers['range']
+ 'bytes=9500-'
+
+ Args:
+ start (Optional[int]): The first byte in a range. Can be zero,
+ positive, negative or :data:`None`.
+ end (Optional[int]): The last byte in a range. Assumed to be
+ positive.
+ headers (Mapping[str, str]): A headers mapping which can have the
+ bytes range added if at least one of ``start`` or ``end``
+ is not :data:`None`.
+ """
+ if start is None:
+ if end is None:
+ # No range to add.
+ return
+ else:
+ # NOTE: This assumes ``end`` is non-negative.
+ bytes_range = "0-{:d}".format(end)
+ else:
+ if end is None:
+ if start < 0:
+ bytes_range = "{:d}".format(start)
+ else:
+ bytes_range = "{:d}-".format(start)
+ else:
+ # NOTE: This is invalid if ``start < 0``.
+ bytes_range = "{:d}-{:d}".format(start, end)
+
+ headers[_helpers.RANGE_HEADER] = "bytes=" + bytes_range
+
+
+def get_range_info(response, get_headers, callback=_helpers.do_nothing):
+ """Get the start, end and total bytes from a content range header.
+
+ Args:
+ response (object): An HTTP response object.
+ get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
+ from an HTTP response.
+ callback (Optional[Callable]): A callback that takes no arguments,
+ to be executed when an exception is being raised.
+
+ Returns:
+ Tuple[int, int, int]: The start byte, end byte and total bytes.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the
+ ``Content-Range`` header is not of the form
+ ``bytes {start}-{end}/{total}``.
+ """
+ content_range = _helpers.header_required(
+ response, _helpers.CONTENT_RANGE_HEADER, get_headers, callback=callback
+ )
+ match = _CONTENT_RANGE_RE.match(content_range)
+ if match is None:
+ callback()
+ raise common.InvalidResponse(
+ response,
+ "Unexpected content-range header",
+ content_range,
+ 'Expected to be of the form "bytes {start}-{end}/{total}"',
+ )
+
+ return (
+ int(match.group("start_byte")),
+ int(match.group("end_byte")),
+ int(match.group("total_bytes")),
+ )
+
+
+def _check_for_zero_content_range(response, get_status_code, get_headers):
+ """Validate if response status code is 416 and content range is zero.
+
+ This is the special case for handling zero bytes files.
+
+ Args:
+ response (object): An HTTP response object.
+ get_status_code (Callable[Any, int]): Helper to get a status code
+ from a response.
+ get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
+ from an HTTP response.
+
+ Returns:
+ bool: True if content range total bytes is zero, false otherwise.
+ """
+ if get_status_code(response) == http.client.REQUESTED_RANGE_NOT_SATISFIABLE:
+ content_range = _helpers.header_required(
+ response,
+ _helpers.CONTENT_RANGE_HEADER,
+ get_headers,
+ callback=_helpers.do_nothing,
+ )
+ if content_range == _ZERO_CONTENT_RANGE_HEADER:
+ return True
+ return False
diff --git a/packages/google-resumable-media/google/_async_resumable_media/_helpers.py b/packages/google-resumable-media/google/_async_resumable_media/_helpers.py
new file mode 100644
index 000000000000..8cf0f2e96acb
--- /dev/null
+++ b/packages/google-resumable-media/google/_async_resumable_media/_helpers.py
@@ -0,0 +1,197 @@
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared utilities used by both downloads and uploads."""
+
+import logging
+import random
+import time
+
+
+from google.resumable_media import common
+
+
+RANGE_HEADER = "range"
+CONTENT_RANGE_HEADER = "content-range"
+
+_SLOW_CRC32C_WARNING = (
+ "Currently using crcmod in pure python form. This is a slow "
+ "implementation. Python 3 has a faster implementation, `google-crc32c`, "
+ "which will be used if it is installed."
+)
+_HASH_HEADER = "x-goog-hash"
+_MISSING_CHECKSUM = """\
+No {checksum_type} checksum was returned from the service while downloading {}
+(which happens for composite objects), so client-side content integrity
+checking is not being performed."""
+_LOGGER = logging.getLogger(__name__)
+
+
+def do_nothing():
+ """Simple default callback."""
+
+
+def header_required(response, name, get_headers, callback=do_nothing):
+ """Checks that a specific header is in a headers dictionary.
+
+ Args:
+ response (object): An HTTP response object, expected to have a
+ ``headers`` attribute that is a ``Mapping[str, str]``.
+ name (str): The name of a required header.
+ get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
+ from an HTTP response.
+ callback (Optional[Callable]): A callback that takes no arguments,
+ to be executed when an exception is being raised.
+
+ Returns:
+ str: The desired header.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the header
+ is missing.
+ """
+ headers = get_headers(response)
+ if name not in headers:
+ callback()
+ raise common.InvalidResponse(
+ response, "Response headers must contain header", name
+ )
+
+ return headers[name]
+
+
+def require_status_code(response, status_codes, get_status_code, callback=do_nothing):
+ """Require a response has a status code among a list.
+
+ Args:
+ response (object): The HTTP response object.
+ status_codes (tuple): The acceptable status codes.
+ get_status_code (Callable[Any, int]): Helper to get a status code
+ from a response.
+ callback (Optional[Callable]): A callback that takes no arguments,
+ to be executed when an exception is being raised.
+
+ Returns:
+ int: The status code.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status code
+ is not one of the values in ``status_codes``.
+ """
+ status_code = get_status_code(response)
+ if status_code not in status_codes:
+ callback()
+ raise common.InvalidResponse(
+ response,
+ "Request failed with status code",
+ status_code,
+ "Expected one of",
+ *status_codes,
+ )
+ return status_code
+
+
+def calculate_retry_wait(base_wait, max_sleep):
+ """Calculate the amount of time to wait before a retry attempt.
+
+ Wait time grows exponentially with the number of attempts, until
+ ``max_sleep``.
+
+ A random amount of jitter (between 0 and 1 seconds) is added to spread out
+ retry attempts from different clients.
+
+ Args:
+ base_wait (float): The "base" wait time (i.e. without any jitter)
+ that will be doubled until it reaches the maximum sleep.
+ max_sleep (float): Maximum value that a sleep time is allowed to be.
+
+ Returns:
+ Tuple[float, float]: The new base wait time as well as the wait time
+ to be applied (with a random amount of jitter between 0 and 1 seconds
+ added).
+ """
+ new_base_wait = 2.0 * base_wait
+ if new_base_wait > max_sleep:
+ new_base_wait = max_sleep
+
+ jitter_ms = random.randint(0, 1000)
+ return new_base_wait, new_base_wait + 0.001 * jitter_ms
+
+
+async def wait_and_retry(func, get_status_code, retry_strategy):
+ """Attempts to retry a call to ``func`` until success.
+
+ Expects ``func`` to return an HTTP response and uses ``get_status_code``
+ to check if the response is retry-able.
+
+ Will retry until :meth:`~.RetryStrategy.retry_allowed` (on the current
+ ``retry_strategy``) returns :data:`False`. Uses
+ :func:`calculate_retry_wait` to double the wait time (with jitter) after
+ each attempt.
+
+ Args:
+ func (Callable): A callable that takes no arguments and produces
+ an HTTP response which will be checked as retry-able.
+ get_status_code (Callable[Any, int]): Helper to get a status code
+ from a response.
+ retry_strategy (~google.resumable_media.common.RetryStrategy): The
+ strategy to use if the request fails and must be retried.
+
+ Returns:
+ object: The return value of ``func``.
+ """
+
+ total_sleep = 0.0
+ num_retries = 0
+ base_wait = 0.5 # When doubled will give 1.0
+
+ while True: # return on success or when retries exhausted.
+ error = None
+ try:
+ response = await func()
+ except ConnectionError as e:
+ error = e
+ else:
+ if get_status_code(response) not in common.RETRYABLE:
+ return response
+
+ if not retry_strategy.retry_allowed(total_sleep, num_retries):
+ # Retries are exhausted and no acceptable response was received. Raise the
+ # retriable_error or return the unacceptable response.
+ if error:
+ raise error
+
+ return response
+
+ base_wait, wait_time = calculate_retry_wait(base_wait, retry_strategy.max_sleep)
+
+ num_retries += 1
+ total_sleep += wait_time
+ time.sleep(wait_time)
+
+
+class _DoNothingHash(object):
+ """Do-nothing hash object.
+
+ Intended as a stand-in for ``hashlib.md5`` or a crc32c checksum
+ implementation in cases where it isn't necessary to compute the hash.
+ """
+
+ def update(self, unused_chunk):
+ """Do-nothing ``update`` method.
+
+ Intended to match the interface of ``hashlib.md5`` and other checksums.
+ Args:
+ unused_chunk (bytes): A chunk of data.
+ """
diff --git a/packages/google-resumable-media/google/_async_resumable_media/_upload.py b/packages/google-resumable-media/google/_async_resumable_media/_upload.py
new file mode 100644
index 000000000000..9f5b0de1b539
--- /dev/null
+++ b/packages/google-resumable-media/google/_async_resumable_media/_upload.py
@@ -0,0 +1,976 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Virtual bases classes for uploading media via Google APIs.
+
+Supported here are:
+
+* simple (media) uploads
+* multipart uploads that contain both metadata and a small file as payload
+* resumable uploads (with metadata as well)
+"""
+
+import http.client
+import json
+import os
+import random
+import sys
+
+from google import _async_resumable_media
+from google._async_resumable_media import _helpers
+from google.resumable_media import _helpers as sync_helpers
+from google.resumable_media import _upload as sync_upload
+from google.resumable_media import common
+
+
+from google.resumable_media._upload import (
+ _CONTENT_TYPE_HEADER,
+ _CONTENT_RANGE_TEMPLATE,
+ _RANGE_UNKNOWN_TEMPLATE,
+ _EMPTY_RANGE_TEMPLATE,
+ _BOUNDARY_FORMAT,
+ _MULTIPART_SEP,
+ _CRLF,
+ _MULTIPART_BEGIN,
+ _RELATED_HEADER,
+ _BYTES_RANGE_RE,
+ _STREAM_ERROR_TEMPLATE,
+ _POST,
+ _PUT,
+ _UPLOAD_CHECKSUM_MISMATCH_MESSAGE,
+ _UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE,
+)
+
+
+class UploadBase(object):
+ """Base class for upload helpers.
+
+ Defines core shared behavior across different upload types.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ def __init__(self, upload_url, headers=None):
+ self.upload_url = upload_url
+ if headers is None:
+ headers = {}
+ self._headers = headers
+ self._finished = False
+ self._retry_strategy = common.RetryStrategy()
+
+ @property
+ def finished(self):
+ """bool: Flag indicating if the upload has completed."""
+ return self._finished
+
+ def _process_response(self, response):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ # Tombstone the current upload so it cannot be used again (in either
+ # failure or success).
+ self._finished = True
+ _helpers.require_status_code(response, (http.client.OK,), self._get_status_code)
+
+ @staticmethod
+ def _get_status_code(response):
+ """Access the status code from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ @staticmethod
+ def _get_headers(response):
+ """Access the headers from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ @staticmethod
+ def _get_body(response):
+ """Access the response body from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class SimpleUpload(UploadBase):
+ """Upload a resource to a Google API.
+
+ A **simple** media upload sends no metadata and completes the upload
+ in a single request.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ def _prepare_request(self, data, content_type):
+ """Prepare the contents of an HTTP request.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ .. note:
+
+ This method will be used only once, so ``headers`` will be
+ mutated by having a new key added to it.
+
+ Args:
+ data (bytes): The resource content to be uploaded.
+ content_type (str): The content type for the request.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the current upload has already finished.
+ TypeError: If ``data`` isn't bytes.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("An upload can only be used once.")
+
+ if not isinstance(data, bytes):
+ raise TypeError("`data` must be bytes, received", type(data))
+ self._headers[_CONTENT_TYPE_HEADER] = content_type
+ return _POST, self.upload_url, data, self._headers
+
+ def transmit(self, transport, data, content_type, timeout=None):
+ """Transmit the resource to be uploaded.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ data (bytes): The resource content to be uploaded.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class MultipartUpload(UploadBase):
+ """Upload a resource with metadata to a Google API.
+
+ A **multipart** upload sends both metadata and the resource in a single
+ (multipart) request.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The request metadata will be amended
+ to include the computed value. Using this option will override a
+ manually-set checksum value. Supported values are "md5", "crc32c"
+ and None. The default is None.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ def __init__(self, upload_url, headers=None, checksum=None):
+ super(MultipartUpload, self).__init__(upload_url, headers=headers)
+ self._checksum_type = checksum
+
+ def _prepare_request(self, data, metadata, content_type):
+ """Prepare the contents of an HTTP request.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ .. note:
+
+ This method will be used only once, so ``headers`` will be
+ mutated by having a new key added to it.
+
+ Args:
+ data (bytes): The resource content to be uploaded.
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the current upload has already finished.
+ TypeError: If ``data`` isn't bytes.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("An upload can only be used once.")
+
+ if not isinstance(data, bytes):
+ raise TypeError("`data` must be bytes, received", type(data))
+
+ checksum_object = sync_helpers._get_checksum_object(self._checksum_type)
+
+ if checksum_object is not None:
+ checksum_object.update(data)
+ actual_checksum = sync_helpers.prepare_checksum_digest(
+ checksum_object.digest()
+ )
+ metadata_key = sync_helpers._get_metadata_key(self._checksum_type)
+ metadata[metadata_key] = actual_checksum
+
+ content, multipart_boundary = construct_multipart_request(
+ data, metadata, content_type
+ )
+ multipart_content_type = _RELATED_HEADER + multipart_boundary + b'"'
+
+ self._headers[_CONTENT_TYPE_HEADER] = multipart_content_type
+
+ return _POST, self.upload_url, content, self._headers
+
+ def transmit(self, transport, data, metadata, content_type, timeout=None):
+ """Transmit the resource to be uploaded.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ data (bytes): The resource content to be uploaded.
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class ResumableUpload(UploadBase, sync_upload.ResumableUpload):
+ """Initiate and fulfill a resumable upload to a Google API.
+
+ A **resumable** upload sends an initial request with the resource metadata
+ and then gets assigned an upload ID / upload URL to send bytes to.
+ Using the upload URL, the upload is then done in chunks (determined by
+ the user) until all bytes have been uploaded.
+
+ Args:
+ upload_url (str): The URL where the resumable upload will be initiated.
+ chunk_size (int): The size of each chunk used to upload the resource.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the :meth:`initiate` request, e.g. headers for
+ encrypted data. These **will not** be sent with
+ :meth:`transmit_next_chunk` or :meth:`recover` requests.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. After the upload is complete, the
+ server-computed checksum of the resulting object will be read
+ and google.resumable_media.common.DataCorruption will be raised on
+ a mismatch. The corrupted file will not be deleted from the remote
+ host automatically. Supported values are "md5", "crc32c" and None.
+ The default is None.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+
+ Raises:
+ ValueError: If ``chunk_size`` is not a multiple of
+ :data:`.UPLOAD_CHUNK_SIZE`.
+ """
+
+ def __init__(self, upload_url, chunk_size, checksum=None, headers=None):
+ super(ResumableUpload, self).__init__(upload_url, headers=headers)
+ if chunk_size % _async_resumable_media.UPLOAD_CHUNK_SIZE != 0:
+ raise ValueError(
+ "{} KB must divide chunk size".format(
+ _async_resumable_media.UPLOAD_CHUNK_SIZE / 1024
+ )
+ )
+ self._chunk_size = chunk_size
+ self._stream = None
+ self._content_type = None
+ self._bytes_uploaded = 0
+ self._bytes_checksummed = 0
+ self._checksum_type = checksum
+ self._checksum_object = None
+ self._total_bytes = None
+ self._resumable_url = None
+ self._invalid = False
+
+ @property
+ def invalid(self):
+ """bool: Indicates if the upload is in an invalid state.
+
+ This will occur if a call to :meth:`transmit_next_chunk` fails.
+ To recover from such a failure, call :meth:`recover`.
+ """
+ return self._invalid
+
+ @property
+ def chunk_size(self):
+ """int: The size of each chunk used to upload the resource."""
+ return self._chunk_size
+
+ @property
+ def resumable_url(self):
+ """Optional[str]: The URL of the in-progress resumable upload."""
+ return self._resumable_url
+
+ @property
+ def bytes_uploaded(self):
+ """int: Number of bytes that have been uploaded."""
+ return self._bytes_uploaded
+
+ @property
+ def total_bytes(self):
+ """Optional[int]: The total number of bytes to be uploaded.
+
+ If this upload is initiated (via :meth:`initiate`) with
+ ``stream_final=True``, this value will be populated based on the size
+ of the ``stream`` being uploaded. (By default ``stream_final=True``.)
+
+ If this upload is initiated with ``stream_final=False``,
+ :attr:`total_bytes` will be :data:`None` since it cannot be
+ determined from the stream.
+ """
+ return self._total_bytes
+
+ def _prepare_initiate_request(
+ self, stream, metadata, content_type, total_bytes=None, stream_final=True
+ ):
+ """Prepare the contents of HTTP request to initiate upload.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ stream (IO[bytes]): The stream (i.e. file-like object) that will
+ be uploaded. The stream **must** be at the beginning (i.e.
+ ``stream.tell() == 0``).
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ total_bytes (Optional[int]): The total number of bytes to be
+ uploaded. If specified, the upload size **will not** be
+ determined from the stream (even if ``stream_final=True``).
+ stream_final (Optional[bool]): Indicates if the ``stream`` is
+ "final" (i.e. no more bytes will be added to it). In this case
+ we determine the upload size from the size of the stream. If
+ ``total_bytes`` is passed, this argument will be ignored.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the current upload has already been initiated.
+ ValueError: If ``stream`` is not at the beginning.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.resumable_url is not None:
+ raise ValueError("This upload has already been initiated.")
+ if stream.tell() != 0:
+ raise ValueError("Stream must be at beginning.")
+
+ self._stream = stream
+ self._content_type = content_type
+ headers = {
+ _CONTENT_TYPE_HEADER: "application/json; charset=UTF-8",
+ "x-upload-content-type": content_type,
+ }
+ # Set the total bytes if possible.
+ if total_bytes is not None:
+ self._total_bytes = total_bytes
+ elif stream_final:
+ self._total_bytes = get_total_bytes(stream)
+ # Add the total bytes to the headers if set.
+ if self._total_bytes is not None:
+ content_length = "{:d}".format(self._total_bytes)
+ headers["x-upload-content-length"] = content_length
+
+ headers.update(self._headers)
+ payload = json.dumps(metadata).encode("utf-8")
+ return _POST, self.upload_url, payload, headers
+
+ def _process_initiate_response(self, response):
+ """Process the response from an HTTP request that initiated upload.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ This method takes the URL from the ``Location`` header and stores it
+ for future use. Within that URL, we assume the ``upload_id`` query
+ parameter has been included, but we do not check.
+
+ Args:
+ response (object): The HTTP response object (need headers).
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ _helpers.require_status_code(
+ response,
+ (http.client.OK,),
+ self._get_status_code,
+ callback=self._make_invalid,
+ )
+ self._resumable_url = _helpers.header_required(
+ response, "location", self._get_headers
+ )
+
+ def initiate(
+ self,
+ transport,
+ stream,
+ metadata,
+ content_type,
+ total_bytes=None,
+ stream_final=True,
+ timeout=None,
+ ):
+ """Initiate a resumable upload.
+
+ By default, this method assumes your ``stream`` is in a "final"
+ state ready to transmit. However, ``stream_final=False`` can be used
+ to indicate that the size of the resource is not known. This can happen
+ if bytes are being dynamically fed into ``stream``, e.g. if the stream
+ is attached to application logs.
+
+ If ``stream_final=False`` is used, :attr:`chunk_size` bytes will be
+ read from the stream every time :meth:`transmit_next_chunk` is called.
+ If one of those reads produces strictly fewer bites than the chunk
+ size, the upload will be concluded.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ stream (IO[bytes]): The stream (i.e. file-like object) that will
+ be uploaded. The stream **must** be at the beginning (i.e.
+ ``stream.tell() == 0``).
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ total_bytes (Optional[int]): The total number of bytes to be
+ uploaded. If specified, the upload size **will not** be
+ determined from the stream (even if ``stream_final=True``).
+ stream_final (Optional[bool]): Indicates if the ``stream`` is
+ "final" (i.e. no more bytes will be added to it). In this case
+ we determine the upload size from the size of the stream. If
+ ``total_bytes`` is passed, this argument will be ignored.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _prepare_request(self):
+ """Prepare the contents of HTTP request to upload a chunk.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O. This is based on the `sans-I/O`_ philosophy.
+
+ For the time being, this **does require** some form of I/O to read
+ a chunk from ``stream`` (via :func:`get_next_chunk`). However, this
+ will (almost) certainly not be network I/O.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always PUT)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ The headers **do not** incorporate the ``_headers`` on the
+ current instance.
+
+ Raises:
+ ValueError: If the current upload has finished.
+ ValueError: If the current upload is in an invalid state.
+ ValueError: If the current upload has not been initiated.
+ ValueError: If the location in the stream (i.e. ``stream.tell()``)
+ does not agree with ``bytes_uploaded``.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("Upload has finished.")
+ if self.invalid:
+ raise ValueError(
+ "Upload is in an invalid state. To recover call `recover()`."
+ )
+ if self.resumable_url is None:
+ raise ValueError(
+ "This upload has not been initiated. Please call "
+ "initiate() before beginning to transmit chunks."
+ )
+
+ start_byte, payload, content_range = get_next_chunk(
+ self._stream, self._chunk_size, self._total_bytes
+ )
+ if start_byte != self.bytes_uploaded:
+ msg = _STREAM_ERROR_TEMPLATE.format(start_byte, self.bytes_uploaded)
+ raise ValueError(msg)
+
+ self._update_checksum(start_byte, payload)
+
+ headers = {
+ _CONTENT_TYPE_HEADER: self._content_type,
+ _helpers.CONTENT_RANGE_HEADER: content_range,
+ }
+ return _PUT, self.resumable_url, payload, headers
+
+ def _make_invalid(self):
+ """Simple setter for ``invalid``.
+
+ This is intended to be passed along as a callback to helpers that
+ raise an exception so they can mark this instance as invalid before
+ raising.
+ """
+ self._invalid = True
+
+ async def _process_resumable_response(self, response, bytes_sent):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+ bytes_sent (int): The number of bytes sent in the request that
+ ``response`` was returned for.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is 308 and the ``range`` header is not of the form
+ ``bytes 0-{end}``.
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200 or 308.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ status_code = _helpers.require_status_code(
+ response,
+ (http.client.OK, http.client.PERMANENT_REDIRECT),
+ self._get_status_code,
+ callback=self._make_invalid,
+ )
+ if status_code == http.client.OK:
+ # NOTE: We use the "local" information of ``bytes_sent`` to update
+ # ``bytes_uploaded``, but do not verify this against other
+ # state. However, there may be some other information:
+ #
+ # * a ``size`` key in JSON response body
+ # * the ``total_bytes`` attribute (if set)
+ # * ``stream.tell()`` (relying on fact that ``initiate()``
+ # requires stream to be at the beginning)
+ self._bytes_uploaded = self._bytes_uploaded + bytes_sent
+ # Tombstone the current upload so it cannot be used again.
+ self._finished = True
+ # Validate the checksum. This can raise an exception on failure.
+ await self._validate_checksum(response)
+ else:
+ bytes_range = _helpers.header_required(
+ response,
+ _helpers.RANGE_HEADER,
+ self._get_headers,
+ callback=self._make_invalid,
+ )
+ match = _BYTES_RANGE_RE.match(bytes_range)
+ if match is None:
+ self._make_invalid()
+ raise common.InvalidResponse(
+ response,
+ 'Unexpected "range" header',
+ bytes_range,
+ 'Expected to be of the form "bytes=0-{end}"',
+ )
+ self._bytes_uploaded = int(match.group("end_byte")) + 1
+
+ async def _validate_checksum(self, response):
+ """Check the computed checksum, if any, against the response headers.
+ Args:
+ response (object): The HTTP response object.
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the checksum
+ computed locally and the checksum reported by the remote host do
+ not match.
+ """
+ if self._checksum_type is None:
+ return
+ metadata_key = sync_helpers._get_metadata_key(self._checksum_type)
+ metadata = await response.json()
+ remote_checksum = metadata.get(metadata_key)
+ if remote_checksum is None:
+ raise common.InvalidResponse(
+ response,
+ _UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(metadata_key),
+ self._get_headers(response),
+ )
+ local_checksum = sync_helpers.prepare_checksum_digest(
+ self._checksum_object.digest()
+ )
+ if local_checksum != remote_checksum:
+ raise common.DataCorruption(
+ response,
+ _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
+ self._checksum_type.upper(), local_checksum, remote_checksum
+ ),
+ )
+
+ def transmit_next_chunk(self, transport, timeout=None):
+ """Transmit the next chunk of the resource to be uploaded.
+
+ If the current upload was initiated with ``stream_final=False``,
+ this method will dynamically determine if the upload has completed.
+ The upload will be considered complete if the stream produces
+ fewer than :attr:`chunk_size` bytes when a chunk is read from it.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _prepare_recover_request(self):
+ """Prepare the contents of HTTP request to recover from failure.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O. This is based on the `sans-I/O`_ philosophy.
+
+ We assume that the :attr:`resumable_url` is set (i.e. the only way
+ the upload can end up :attr:`invalid` is if it has been initiated.
+
+ Returns:
+ Tuple[str, str, NoneType, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always PUT)
+ * the URL for the request
+ * the body of the request (always :data:`None`)
+ * headers for the request
+
+ The headers **do not** incorporate the ``_headers`` on the
+ current instance.
+
+ Raises:
+ ValueError: If the current upload is not in an invalid state.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if not self.invalid:
+ raise ValueError("Upload is not in invalid state, no need to recover.")
+
+ headers = {_helpers.CONTENT_RANGE_HEADER: "bytes */*"}
+ return _PUT, self.resumable_url, None, headers
+
+ def _process_recover_response(self, response):
+ """Process the response from an HTTP request to recover from failure.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 308.
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is 308 and the ``range`` header is not of the form
+ ``bytes 0-{end}``.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ _helpers.require_status_code(
+ response,
+ (http.client.PERMANENT_REDIRECT,),
+ self._get_status_code,
+ )
+ headers = self._get_headers(response)
+ if _helpers.RANGE_HEADER in headers:
+ bytes_range = headers[_helpers.RANGE_HEADER]
+ match = _BYTES_RANGE_RE.match(bytes_range)
+ if match is None:
+ raise common.InvalidResponse(
+ response,
+ 'Unexpected "range" header',
+ bytes_range,
+ 'Expected to be of the form "bytes=0-{end}"',
+ )
+ self._bytes_uploaded = int(match.group("end_byte")) + 1
+ else:
+ # In this case, the upload has not "begun".
+ self._bytes_uploaded = 0
+
+ self._stream.seek(self._bytes_uploaded)
+ self._invalid = False
+
+ def recover(self, transport):
+ """Recover from a failure.
+
+ This method should be used when a :class:`ResumableUpload` is in an
+ :attr:`~ResumableUpload.invalid` state due to a request failure.
+
+ This will verify the progress with the server and make sure the
+ current upload is in a valid state before :meth:`transmit_next_chunk`
+ can be used again.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+def get_boundary():
+ """Get a random boundary for a multipart request.
+
+ Returns:
+ bytes: The boundary used to separate parts of a multipart request.
+ """
+ random_int = random.randrange(sys.maxsize)
+ boundary = _BOUNDARY_FORMAT.format(random_int)
+ # NOTE: Neither % formatting nor .format() are available for byte strings
+ # in Python 3.4, so we must use unicode strings as templates.
+ return boundary.encode("utf-8")
+
+
+def construct_multipart_request(data, metadata, content_type):
+ """Construct a multipart request body.
+
+ Args:
+ data (bytes): The resource content (UTF-8 encoded as bytes)
+ to be uploaded.
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+
+ Returns:
+ Tuple[bytes, bytes]: The multipart request body and the boundary used
+ between each part.
+ """
+ multipart_boundary = get_boundary()
+ json_bytes = json.dumps(metadata).encode("utf-8")
+ content_type = content_type.encode("utf-8")
+ # Combine the two parts into a multipart payload.
+ # NOTE: We'd prefer a bytes template but are restricted by Python 3.4.
+ boundary_sep = _MULTIPART_SEP + multipart_boundary
+ content = (
+ boundary_sep
+ + _MULTIPART_BEGIN
+ + json_bytes
+ + _CRLF
+ + boundary_sep
+ + _CRLF
+ + b"content-type: "
+ + content_type
+ + _CRLF
+ + _CRLF
+ + data # Empty line between headers and body.
+ + _CRLF
+ + boundary_sep
+ + _MULTIPART_SEP
+ )
+
+ return content, multipart_boundary
+
+
+def get_total_bytes(stream):
+ """Determine the total number of bytes in a stream.
+
+ Args:
+ stream (IO[bytes]): The stream (i.e. file-like object).
+
+ Returns:
+ int: The number of bytes.
+ """
+ current_position = stream.tell()
+ # NOTE: ``.seek()`` **should** return the same value that ``.tell()``
+ # returns, but in Python 2, ``file`` objects do not.
+ stream.seek(0, os.SEEK_END)
+ end_position = stream.tell()
+ # Go back to the initial position.
+ stream.seek(current_position)
+
+ return end_position
+
+
+def get_next_chunk(stream, chunk_size, total_bytes):
+ """Get a chunk from an I/O stream.
+
+ The ``stream`` may have fewer bytes remaining than ``chunk_size``
+ so it may not always be the case that
+ ``end_byte == start_byte + chunk_size - 1``.
+
+ Args:
+ stream (IO[bytes]): The stream (i.e. file-like object).
+ chunk_size (int): The size of the chunk to be read from the ``stream``.
+ total_bytes (Optional[int]): The (expected) total number of bytes
+ in the ``stream``.
+
+ Returns:
+ Tuple[int, bytes, str]: Triple of:
+
+ * the start byte index
+ * the content in between the start and end bytes (inclusive)
+ * content range header for the chunk (slice) that has been read
+
+ Raises:
+ ValueError: If ``total_bytes == 0`` but ``stream.read()`` yields
+ non-empty content.
+ ValueError: If there is no data left to consume. This corresponds
+ exactly to the case ``end_byte < start_byte``, which can only
+ occur if ``end_byte == start_byte - 1``.
+ """
+ start_byte = stream.tell()
+ if total_bytes is not None and start_byte + chunk_size >= total_bytes > 0:
+ payload = stream.read(total_bytes - start_byte)
+ else:
+ payload = stream.read(chunk_size)
+ end_byte = stream.tell() - 1
+
+ num_bytes_read = len(payload)
+ if total_bytes is None:
+ if num_bytes_read < chunk_size:
+ # We now **KNOW** the total number of bytes.
+ total_bytes = end_byte + 1
+ elif total_bytes == 0:
+ # NOTE: We also expect ``start_byte == 0`` here but don't check
+ # because ``_prepare_initiate_request()`` requires the
+ # stream to be at the beginning.
+ if num_bytes_read != 0:
+ raise ValueError(
+ "Stream specified as empty, but produced non-empty content."
+ )
+ else:
+ if num_bytes_read == 0:
+ raise ValueError(
+ "Stream is already exhausted. There is no content remaining."
+ )
+
+ content_range = get_content_range(start_byte, end_byte, total_bytes)
+ return start_byte, payload, content_range
+
+
+def get_content_range(start_byte, end_byte, total_bytes):
+ """Convert start, end and total into content range header.
+
+ If ``total_bytes`` is not known, uses "bytes {start}-{end}/*".
+ If we are dealing with an empty range (i.e. ``end_byte < start_byte``)
+ then "bytes */{total}" is used.
+
+ This function **ASSUMES** that if the size is not known, the caller will
+ not also pass an empty range.
+
+ Args:
+ start_byte (int): The start (inclusive) of the byte range.
+ end_byte (int): The end (inclusive) of the byte range.
+ total_bytes (Optional[int]): The number of bytes in the byte
+ range (if known).
+
+ Returns:
+ str: The content range header.
+ """
+ if total_bytes is None:
+ return _RANGE_UNKNOWN_TEMPLATE.format(start_byte, end_byte)
+ elif end_byte < start_byte:
+ return _EMPTY_RANGE_TEMPLATE.format(total_bytes)
+ else:
+ return _CONTENT_RANGE_TEMPLATE.format(start_byte, end_byte, total_bytes)
diff --git a/packages/google-resumable-media/google/_async_resumable_media/requests/__init__.py b/packages/google-resumable-media/google/_async_resumable_media/requests/__init__.py
new file mode 100644
index 000000000000..e6a6190c17f5
--- /dev/null
+++ b/packages/google-resumable-media/google/_async_resumable_media/requests/__init__.py
@@ -0,0 +1,683 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""``requests`` utilities for Google Media Downloads and Resumable Uploads.
+
+This sub-package assumes callers will use the `requests`_ library
+as transport and `google-auth`_ for sending authenticated HTTP traffic
+with ``requests``.
+
+.. _requests: http://docs.python-requests.org/
+.. _google-auth: https://google-auth.readthedocs.io/
+
+====================
+Authorized Transport
+====================
+
+To use ``google-auth`` and ``requests`` to create an authorized transport
+that has read-only access to Google Cloud Storage (GCS):
+
+.. testsetup:: get-credentials
+
+ import google.auth
+ import google.auth.credentials as creds_mod
+ import mock
+
+ def mock_default(scopes=None):
+ credentials = mock.Mock(spec=creds_mod.Credentials)
+ return credentials, 'mock-project'
+
+ # Patch the ``default`` function on the module.
+ original_default = google.auth.default
+ google.auth.default = mock_default
+
+.. doctest:: get-credentials
+
+ >>> import google.auth
+ >>> import google.auth.transport.requests as tr_requests
+ >>>
+ >>> ro_scope = 'https://www.googleapis.com/auth/devstorage.read_only'
+ >>> credentials, _ = google.auth.default(scopes=(ro_scope,))
+ >>> transport = tr_requests.AuthorizedSession(credentials)
+ >>> transport
+
+
+.. testcleanup:: get-credentials
+
+ # Put back the correct ``default`` function on the module.
+ google.auth.default = original_default
+
+================
+Simple Downloads
+================
+
+To download an object from Google Cloud Storage, construct the media URL
+for the GCS object and download it with an authorized transport that has
+access to the resource:
+
+.. testsetup:: basic-download
+
+ import mock
+ import requests
+ import http.client
+
+ bucket = 'bucket-foo'
+ blob_name = 'file.txt'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response.headers['Content-Length'] = '1364156'
+ fake_content = mock.MagicMock(spec=['__len__'])
+ fake_content.__len__.return_value = 1364156
+ fake_response._content = fake_content
+
+ get_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=get_method, spec=['request'])
+
+.. doctest:: basic-download
+
+ >>> from google.resumable_media.requests import Download
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/download/storage/v1/b/'
+ ... '{bucket}/o/{blob_name}?alt=media')
+ >>> media_url = url_template.format(
+ ... bucket=bucket, blob_name=blob_name)
+ >>>
+ >>> download = Download(media_url)
+ >>> response = download.consume(transport)
+ >>> download.finished
+ True
+ >>> response
+
+ >>> response.headers['Content-Length']
+ '1364156'
+ >>> len(response.content)
+ 1364156
+
+To download only a portion of the bytes in the object,
+specify ``start`` and ``end`` byte positions (both optional):
+
+.. testsetup:: basic-download-with-slice
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import Download
+
+ media_url = 'http://test.invalid'
+ start = 4096
+ end = 8191
+ slice_size = end - start + 1
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.PARTIAL_CONTENT)
+ fake_response.headers['Content-Length'] = '{:d}'.format(slice_size)
+ content_range = 'bytes {:d}-{:d}/1364156'.format(start, end)
+ fake_response.headers['Content-Range'] = content_range
+ fake_content = mock.MagicMock(spec=['__len__'])
+ fake_content.__len__.return_value = slice_size
+ fake_response._content = fake_content
+
+ get_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=get_method, spec=['request'])
+
+.. doctest:: basic-download-with-slice
+
+ >>> download = Download(media_url, start=4096, end=8191)
+ >>> response = download.consume(transport)
+ >>> download.finished
+ True
+ >>> response
+
+ >>> response.headers['Content-Length']
+ '4096'
+ >>> response.headers['Content-Range']
+ 'bytes 4096-8191/1364156'
+ >>> len(response.content)
+ 4096
+
+=================
+Chunked Downloads
+=================
+
+For very large objects or objects of unknown size, it may make more sense
+to download the object in chunks rather than all at once. This can be done
+to avoid dropped connections with a poor internet connection or can allow
+multiple chunks to be downloaded in parallel to speed up the total
+download.
+
+A :class:`.ChunkedDownload` uses the same media URL and authorized
+transport that a basic :class:`.Download` would use, but also
+requires a chunk size and a write-able byte ``stream``. The chunk size is used
+to determine how much of the resouce to consume with each request and the
+stream is to allow the resource to be written out (e.g. to disk) without
+having to fit in memory all at once.
+
+.. testsetup:: chunked-download
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ media_url = 'http://test.invalid'
+
+ fifty_mb = 50 * 1024 * 1024
+ one_gb = 1024 * 1024 * 1024
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.PARTIAL_CONTENT)
+ fake_response.headers['Content-Length'] = '{:d}'.format(fifty_mb)
+ content_range = 'bytes 0-{:d}/{:d}'.format(fifty_mb - 1, one_gb)
+ fake_response.headers['Content-Range'] = content_range
+ fake_content_begin = b'The beginning of the chunk...'
+ fake_content = fake_content_begin + b'1' * (fifty_mb - 29)
+ fake_response._content = fake_content
+
+ get_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=get_method, spec=['request'])
+
+.. doctest:: chunked-download
+
+ >>> from google.resumable_media.requests import ChunkedDownload
+ >>>
+ >>> chunk_size = 50 * 1024 * 1024 # 50MB
+ >>> stream = io.BytesIO()
+ >>> download = ChunkedDownload(
+ ... media_url, chunk_size, stream)
+ >>> # Check the state of the download before starting.
+ >>> download.bytes_downloaded
+ 0
+ >>> download.total_bytes is None
+ True
+ >>> response = download.consume_next_chunk(transport)
+ >>> # Check the state of the download after consuming one chunk.
+ >>> download.finished
+ False
+ >>> download.bytes_downloaded # chunk_size
+ 52428800
+ >>> download.total_bytes # 1GB
+ 1073741824
+ >>> response
+
+ >>> response.headers['Content-Length']
+ '52428800'
+ >>> response.headers['Content-Range']
+ 'bytes 0-52428799/1073741824'
+ >>> len(response.content) == chunk_size
+ True
+ >>> stream.seek(0)
+ 0
+ >>> stream.read(29)
+ b'The beginning of the chunk...'
+
+The download will change it's ``finished`` status to :data:`True`
+once the final chunk is consumed. In some cases, the final chunk may
+not be the same size as the other chunks:
+
+.. testsetup:: chunked-download-end
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import ChunkedDownload
+
+ media_url = 'http://test.invalid'
+
+ fifty_mb = 50 * 1024 * 1024
+ one_gb = 1024 * 1024 * 1024
+ stream = mock.Mock(spec=['write'])
+ download = ChunkedDownload(media_url, fifty_mb, stream)
+ download._bytes_downloaded = 20 * fifty_mb
+ download._total_bytes = one_gb
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.PARTIAL_CONTENT)
+ slice_size = one_gb - 20 * fifty_mb
+ fake_response.headers['Content-Length'] = '{:d}'.format(slice_size)
+ content_range = 'bytes {:d}-{:d}/{:d}'.format(
+ 20 * fifty_mb, one_gb - 1, one_gb)
+ fake_response.headers['Content-Range'] = content_range
+ fake_content = mock.MagicMock(spec=['__len__'])
+ fake_content.__len__.return_value = slice_size
+ fake_response._content = fake_content
+
+ get_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=get_method, spec=['request'])
+
+.. doctest:: chunked-download-end
+
+ >>> # The state of the download in progress.
+ >>> download.finished
+ False
+ >>> download.bytes_downloaded # 20 chunks at 50MB
+ 1048576000
+ >>> download.total_bytes # 1GB
+ 1073741824
+ >>> response = download.consume_next_chunk(transport)
+ >>> # The state of the download after consuming the final chunk.
+ >>> download.finished
+ True
+ >>> download.bytes_downloaded == download.total_bytes
+ True
+ >>> response
+
+ >>> response.headers['Content-Length']
+ '25165824'
+ >>> response.headers['Content-Range']
+ 'bytes 1048576000-1073741823/1073741824'
+ >>> len(response.content) < download.chunk_size
+ True
+
+In addition, a :class:`.ChunkedDownload` can also take optional
+``start`` and ``end`` byte positions.
+
+Usually, no checksum is returned with a chunked download. Even if one is returned,
+it is not validated. If you need to validate the checksum, you can do so
+by buffering the chunks and validating the checksum against the completed download.
+
+==============
+Simple Uploads
+==============
+
+Among the three supported upload classes, the simplest is
+:class:`.SimpleUpload`. A simple upload should be used when the resource
+being uploaded is small and when there is no metadata (other than the name)
+associated with the resource.
+
+.. testsetup:: simple-upload
+
+ import json
+
+ import mock
+ import requests
+ import http.client
+
+ bucket = 'some-bucket'
+ blob_name = 'file.txt'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ payload = {
+ 'bucket': bucket,
+ 'contentType': 'text/plain',
+ 'md5Hash': 'M0XLEsX9/sMdiI+4pB4CAQ==',
+ 'name': blob_name,
+ 'size': '27',
+ }
+ fake_response._content = json.dumps(payload).encode('utf-8')
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+.. doctest:: simple-upload
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> from google.resumable_media.requests import SimpleUpload
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?'
+ ... 'uploadType=media&'
+ ... 'name={blob_name}')
+ >>> upload_url = url_template.format(
+ ... bucket=bucket, blob_name=blob_name)
+ >>>
+ >>> upload = SimpleUpload(upload_url)
+ >>> data = b'Some not too large content.'
+ >>> content_type = 'text/plain'
+ >>> response = upload.transmit(transport, data, content_type)
+ >>> upload.finished
+ True
+ >>> response
+
+ >>> json_response = response.json()
+ >>> json_response['bucket'] == bucket
+ True
+ >>> json_response['name'] == blob_name
+ True
+ >>> json_response['contentType'] == content_type
+ True
+ >>> json_response['md5Hash']
+ 'M0XLEsX9/sMdiI+4pB4CAQ=='
+ >>> int(json_response['size']) == len(data)
+ True
+
+In the rare case that an upload fails, an :exc:`.InvalidResponse`
+will be raised:
+
+.. testsetup:: simple-upload-fail
+
+ import time
+
+ import mock
+ import requests
+ import http.client
+
+ from google import resumable_media
+ from google.resumable_media import _helpers
+ from google.resumable_media.requests import SimpleUpload as constructor
+
+ upload_url = 'http://test.invalid'
+ data = b'Some not too large content.'
+ content_type = 'text/plain'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.SERVICE_UNAVAILABLE)
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+ time_sleep = time.sleep
+ def dont_sleep(seconds):
+ raise RuntimeError('No sleep', seconds)
+
+ def SimpleUpload(*args, **kwargs):
+ upload = constructor(*args, **kwargs)
+ # Mock the cumulative sleep to avoid retries (and `time.sleep()`).
+ upload._retry_strategy = resumable_media.RetryStrategy(
+ max_cumulative_retry=-1.0)
+ return upload
+
+ time.sleep = dont_sleep
+
+.. doctest:: simple-upload-fail
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> upload = SimpleUpload(upload_url)
+ >>> error = None
+ >>> try:
+ ... upload.transmit(transport, data, content_type)
+ ... except resumable_media.InvalidResponse as caught_exc:
+ ... error = caught_exc
+ ...
+ >>> error
+ InvalidResponse('Request failed with status code', 503,
+ 'Expected one of', )
+ >>> error.response
+
+ >>>
+ >>> upload.finished
+ True
+
+.. testcleanup:: simple-upload-fail
+
+ # Put back the correct ``sleep`` function on the ``time`` module.
+ time.sleep = time_sleep
+
+Even in the case of failure, we see that the upload is
+:attr:`~.SimpleUpload.finished`, i.e. it cannot be re-used.
+
+=================
+Multipart Uploads
+=================
+
+After the simple upload, the :class:`.MultipartUpload` can be used to
+achieve essentially the same task. However, a multipart upload allows some
+metadata about the resource to be sent along as well. (This is the "multi":
+we send a first part with the metadata and a second part with the actual
+bytes in the resource.)
+
+Usage is similar to the simple upload, but :meth:`~.MultipartUpload.transmit`
+accepts an extra required argument: ``metadata``.
+
+.. testsetup:: multipart-upload
+
+ import json
+
+ import mock
+ import requests
+ import http.client
+
+ bucket = 'some-bucket'
+ blob_name = 'file.txt'
+ data = b'Some not too large content.'
+ content_type = 'text/plain'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ payload = {
+ 'bucket': bucket,
+ 'name': blob_name,
+ 'metadata': {'color': 'grurple'},
+ }
+ fake_response._content = json.dumps(payload).encode('utf-8')
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+.. doctest:: multipart-upload
+
+ >>> from google.resumable_media.requests import MultipartUpload
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?'
+ ... 'uploadType=multipart')
+ >>> upload_url = url_template.format(bucket=bucket)
+ >>>
+ >>> upload = MultipartUpload(upload_url)
+ >>> metadata = {
+ ... 'name': blob_name,
+ ... 'metadata': {
+ ... 'color': 'grurple',
+ ... },
+ ... }
+ >>> response = upload.transmit(transport, data, metadata, content_type)
+ >>> upload.finished
+ True
+ >>> response
+
+ >>> json_response = response.json()
+ >>> json_response['bucket'] == bucket
+ True
+ >>> json_response['name'] == blob_name
+ True
+ >>> json_response['metadata'] == metadata['metadata']
+ True
+
+As with the simple upload, in the case of failure an :exc:`.InvalidResponse`
+is raised, enclosing the :attr:`~.InvalidResponse.response` that caused
+the failure and the ``upload`` object cannot be re-used after a failure.
+
+=================
+Resumable Uploads
+=================
+
+A :class:`.ResumableUpload` deviates from the other two upload classes:
+it transmits a resource over the course of multiple requests. This
+is intended to be used in cases where:
+
+* the size of the resource is not known (i.e. it is generated on the fly)
+* requests must be short-lived
+* the client has request **size** limitations
+* the resource is too large to fit into memory
+
+In general, a resource should be sent in a **single** request to avoid
+latency and reduce QPS. See `GCS best practices`_ for more things to
+consider when using a resumable upload.
+
+.. _GCS best practices: https://cloud.google.com/storage/docs/\
+ best-practices#uploading
+
+After creating a :class:`.ResumableUpload` instance, a
+**resumable upload session** must be initiated to let the server know that
+a series of chunked upload requests will be coming and to obtain an
+``upload_id`` for the session. In contrast to the other two upload classes,
+:meth:`~.ResumableUpload.initiate` takes a byte ``stream`` as input rather
+than raw bytes as ``data``. This can be a file object, a :class:`~io.BytesIO`
+object or any other stream implementing the same interface.
+
+.. testsetup:: resumable-initiate
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ bucket = 'some-bucket'
+ blob_name = 'file.txt'
+ data = b'Some resumable bytes.'
+ content_type = 'text/plain'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response._content = b''
+ upload_id = 'ABCdef189XY_super_serious'
+ resumable_url_template = (
+ 'https://www.googleapis.com/upload/storage/v1/b/{bucket}'
+ '/o?uploadType=resumable&upload_id={upload_id}')
+ resumable_url = resumable_url_template.format(
+ bucket=bucket, upload_id=upload_id)
+ fake_response.headers['location'] = resumable_url
+ fake_response.headers['x-guploader-uploadid'] = upload_id
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+.. doctest:: resumable-initiate
+
+ >>> from google.resumable_media.requests import ResumableUpload
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?'
+ ... 'uploadType=resumable')
+ >>> upload_url = url_template.format(bucket=bucket)
+ >>>
+ >>> chunk_size = 1024 * 1024 # 1MB
+ >>> upload = ResumableUpload(upload_url, chunk_size)
+ >>> stream = io.BytesIO(data)
+ >>> # The upload doesn't know how "big" it is until seeing a stream.
+ >>> upload.total_bytes is None
+ True
+ >>> metadata = {'name': blob_name}
+ >>> response = upload.initiate(transport, stream, metadata, content_type)
+ >>> response
+
+ >>> upload.resumable_url == response.headers['Location']
+ True
+ >>> upload.total_bytes == len(data)
+ True
+ >>> upload_id = response.headers['X-GUploader-UploadID']
+ >>> upload_id
+ 'ABCdef189XY_super_serious'
+ >>> upload.resumable_url == upload_url + '&upload_id=' + upload_id
+ True
+
+Once a :class:`.ResumableUpload` has been initiated, the resource is
+transmitted in chunks until completion:
+
+.. testsetup:: resumable-transmit
+
+ import io
+ import json
+
+ import mock
+ import requests
+ import http.client
+
+ from google import resumable_media
+ import google.resumable_media.requests.upload as upload_mod
+
+ data = b'01234567891'
+ stream = io.BytesIO(data)
+ # Create an "already initiated" upload.
+ upload_url = 'http://test.invalid'
+ chunk_size = 256 * 1024 # 256KB
+ upload = upload_mod.ResumableUpload(upload_url, chunk_size)
+ upload._resumable_url = 'http://test.invalid?upload_id=mocked'
+ upload._stream = stream
+ upload._content_type = 'text/plain'
+ upload._total_bytes = len(data)
+
+ # After-the-fact update the chunk size so that len(data)
+ # is split into three.
+ upload._chunk_size = 4
+ # Make three fake responses.
+ fake_response0 = requests.Response()
+ fake_response0.status_code = http.client.PERMANENT_REDIRECT
+ fake_response0.headers['range'] = 'bytes=0-3'
+
+ fake_response1 = requests.Response()
+ fake_response1.status_code = http.client.PERMANENT_REDIRECT
+ fake_response1.headers['range'] = 'bytes=0-7'
+
+ fake_response2 = requests.Response()
+ fake_response2.status_code = int(http.client.OK)
+ bucket = 'some-bucket'
+ blob_name = 'file.txt'
+ payload = {
+ 'bucket': bucket,
+ 'name': blob_name,
+ 'size': '{:d}'.format(len(data)),
+ }
+ fake_response2._content = json.dumps(payload).encode('utf-8')
+
+ # Use the fake responses to mock a transport.
+ responses = [fake_response0, fake_response1, fake_response2]
+ put_method = mock.Mock(side_effect=responses, spec=[])
+ transport = mock.Mock(request=put_method, spec=['request'])
+
+.. doctest:: resumable-transmit
+
+ >>> response0 = upload.transmit_next_chunk(transport)
+ >>> response0
+
+ >>> upload.finished
+ False
+ >>> upload.bytes_uploaded == upload.chunk_size
+ True
+ >>>
+ >>> response1 = upload.transmit_next_chunk(transport)
+ >>> response1
+
+ >>> upload.finished
+ False
+ >>> upload.bytes_uploaded == 2 * upload.chunk_size
+ True
+ >>>
+ >>> response2 = upload.transmit_next_chunk(transport)
+ >>> response2
+
+ >>> upload.finished
+ True
+ >>> upload.bytes_uploaded == upload.total_bytes
+ True
+ >>> json_response = response2.json()
+ >>> json_response['bucket'] == bucket
+ True
+ >>> json_response['name'] == blob_name
+ True
+"""
+
+from google._async_resumable_media.requests.download import ChunkedDownload
+from google._async_resumable_media.requests.download import Download
+from google._async_resumable_media.requests.upload import MultipartUpload
+from google._async_resumable_media.requests.download import RawChunkedDownload
+from google._async_resumable_media.requests.download import RawDownload
+from google._async_resumable_media.requests.upload import ResumableUpload
+from google._async_resumable_media.requests.upload import SimpleUpload
+
+
+__all__ = [
+ "ChunkedDownload",
+ "Download",
+ "MultipartUpload",
+ "RawChunkedDownload",
+ "RawDownload",
+ "ResumableUpload",
+ "SimpleUpload",
+]
diff --git a/packages/google-resumable-media/google/_async_resumable_media/requests/_request_helpers.py b/packages/google-resumable-media/google/_async_resumable_media/requests/_request_helpers.py
new file mode 100644
index 000000000000..cd9b9b85b3ef
--- /dev/null
+++ b/packages/google-resumable-media/google/_async_resumable_media/requests/_request_helpers.py
@@ -0,0 +1,154 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared utilities used by both downloads and uploads.
+
+This utilities are explicitly catered to ``requests``-like transports.
+"""
+
+import functools
+
+from google._async_resumable_media import _helpers
+from google.resumable_media import common
+
+from google.auth.transport import _aiohttp_requests as aiohttp_requests # type: ignore
+import aiohttp # type: ignore
+
+_DEFAULT_RETRY_STRATEGY = common.RetryStrategy()
+_SINGLE_GET_CHUNK_SIZE = 8192
+
+
+# The number of seconds to wait to establish a connection
+# (connect() call on socket). Avoid setting this to a multiple of 3 to not
+# Align with TCP Retransmission timing. (typically 2.5-3s)
+_DEFAULT_CONNECT_TIMEOUT = 61
+# The number of seconds to wait between bytes sent from the server.
+_DEFAULT_READ_TIMEOUT = 60
+_DEFAULT_TIMEOUT = aiohttp.ClientTimeout(
+ connect=_DEFAULT_CONNECT_TIMEOUT, sock_read=_DEFAULT_READ_TIMEOUT
+)
+
+
+class RequestsMixin(object):
+ """Mix-in class implementing ``requests``-specific behavior.
+
+ These are methods that are more general purpose, with implementations
+ specific to the types defined in ``requests``.
+ """
+
+ @staticmethod
+ def _get_status_code(response):
+ """Access the status code from an HTTP response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Returns:
+ int: The status code.
+ """
+ return response.status
+
+ @staticmethod
+ def _get_headers(response):
+ """Access the headers from an HTTP response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Returns:
+ ~requests.structures.CaseInsensitiveDict: The header mapping (keys
+ are case-insensitive).
+ """
+ # For Async testing,`_headers` is modified instead of headers
+ # access via the internal field.
+ return response._headers
+
+ @staticmethod
+ async def _get_body(response):
+ """Access the response body from an HTTP response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Returns:
+ bytes: The body of the ``response``.
+ """
+ wrapped_response = aiohttp_requests._CombinedResponse(response)
+ content = await wrapped_response.data.read()
+ return content
+
+
+class RawRequestsMixin(RequestsMixin):
+ @staticmethod
+ async def _get_body(response):
+ """Access the response body from an HTTP response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Returns:
+ bytes: The body of the ``response``.
+ """
+
+ wrapped_response = aiohttp_requests._CombinedResponse(response)
+ content = await wrapped_response.raw_content()
+ return content
+
+
+async def http_request(
+ transport,
+ method,
+ url,
+ data=None,
+ headers=None,
+ retry_strategy=_DEFAULT_RETRY_STRATEGY,
+ **transport_kwargs,
+):
+ """Make an HTTP request.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can make
+ authenticated requests via a ``request()`` method. This method
+ must accept an HTTP method, an upload URL, a ``data`` keyword
+ argument and a ``headers`` keyword argument.
+ method (str): The HTTP method for the request.
+ url (str): The URL for the request.
+ data (Optional[bytes]): The body of the request.
+ headers (Mapping[str, str]): The headers for the request (``transport``
+ may also add additional headers).
+ retry_strategy (~google.resumable_media.common.RetryStrategy): The
+ strategy to use if the request fails and must be retried.
+ transport_kwargs (Dict[str, str]): Extra keyword arguments to be
+ passed along to ``transport.request``.
+
+ Returns:
+ ~requests.Response: The return value of ``transport.request()``.
+ """
+
+ # NOTE(asyncio/aiohttp): Sync versions use a tuple for two timeouts,
+ # default connect timeout and read timeout. Since async requests only
+ # accepts a single value, this is using the connect timeout. This logic
+ # diverges from the sync implementation.
+ if "timeout" not in transport_kwargs:
+ timeout = _DEFAULT_TIMEOUT
+ transport_kwargs["timeout"] = timeout
+
+ func = functools.partial(
+ transport.request, method, url, data=data, headers=headers, **transport_kwargs
+ )
+
+ resp = await _helpers.wait_and_retry(
+ func, RequestsMixin._get_status_code, retry_strategy
+ )
+ return resp
diff --git a/packages/google-resumable-media/google/_async_resumable_media/requests/download.py b/packages/google-resumable-media/google/_async_resumable_media/requests/download.py
new file mode 100644
index 000000000000..490017cf2880
--- /dev/null
+++ b/packages/google-resumable-media/google/_async_resumable_media/requests/download.py
@@ -0,0 +1,469 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Support for downloading media from Google APIs."""
+
+import urllib3.response # type: ignore
+import http
+
+from google._async_resumable_media import _download
+from google._async_resumable_media import _helpers
+from google._async_resumable_media.requests import _request_helpers
+from google.resumable_media import common
+from google.resumable_media import _helpers as sync_helpers
+from google.resumable_media.requests import download
+
+_CHECKSUM_MISMATCH = download._CHECKSUM_MISMATCH
+
+
+class Download(_request_helpers.RequestsMixin, _download.Download):
+ """Helper to manage downloading a resource from a Google API.
+
+ "Slices" of the resource can be retrieved by specifying a range
+ with ``start`` and / or ``end``. However, in typical usage, neither
+ ``start`` nor ``end`` is expected to be provided.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ the downloaded resource can be written to.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, but ``end`` is provided, will download from the
+ beginning to ``end`` of the media.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, but ``start`` is provided, will download from the
+ ``start`` to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The response headers must contain
+ a checksum of the requested type. If the headers lack an
+ appropriate checksum (for instance in the case of transcoded or
+ ranged downloads where the remote service does not know the
+ correct checksum) an INFO-level log will be emitted. Supported
+ values are "md5", "crc32c" and None. The default is "md5".
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ """
+
+ async def _write_to_stream(self, response):
+ """Write response body to a write-able stream.
+
+ .. note:
+
+ This method assumes that the ``_stream`` attribute is set on the
+ current download.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the download's
+ checksum doesn't agree with server-computed checksum.
+ """
+
+ # `_get_expected_checksum()` may return None even if a checksum was
+ # requested, in which case it will emit an info log _MISSING_CHECKSUM.
+ # If an invalid checksum type is specified, this will raise ValueError.
+ expected_checksum, checksum_object = sync_helpers._get_expected_checksum(
+ response, self._get_headers, self.media_url, checksum_type=self.checksum
+ )
+
+ local_checksum_object = _add_decoder(response, checksum_object)
+
+ async for chunk in response.content.iter_chunked(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE
+ ):
+ self._stream.write(chunk)
+ local_checksum_object.update(chunk)
+
+ # Don't validate the checksum for partial responses.
+ if (
+ expected_checksum is not None
+ and response.status != http.client.PARTIAL_CONTENT
+ ):
+ actual_checksum = sync_helpers.prepare_checksum_digest(
+ checksum_object.digest()
+ )
+ if actual_checksum != expected_checksum:
+ msg = _CHECKSUM_MISMATCH.format(
+ self.media_url,
+ expected_checksum,
+ actual_checksum,
+ checksum_type=self.checksum.upper(),
+ )
+ raise common.DataCorruption(response, msg)
+
+ async def consume(self, transport, timeout=_request_helpers._DEFAULT_TIMEOUT):
+ """Consume the resource to be downloaded.
+
+ If a ``stream`` is attached to this download, then the downloaded
+ resource will be written to the stream.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the download's
+ checksum doesn't agree with server-computed checksum.
+ ValueError: If the current :class:`Download` has already
+ finished.
+ """
+ method, url, payload, headers = self._prepare_request()
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ request_kwargs = {
+ "data": payload,
+ "headers": headers,
+ "retry_strategy": self._retry_strategy,
+ "timeout": timeout,
+ }
+
+ if self._stream is not None:
+ request_kwargs["stream"] = True
+
+ result = await _request_helpers.http_request(
+ transport, method, url, **request_kwargs
+ )
+
+ self._process_response(result)
+
+ if self._stream is not None:
+ await self._write_to_stream(result)
+
+ return result
+
+
+class RawDownload(_request_helpers.RawRequestsMixin, _download.Download):
+ """Helper to manage downloading a raw resource from a Google API.
+
+ "Slices" of the resource can be retrieved by specifying a range
+ with ``start`` and / or ``end``. However, in typical usage, neither
+ ``start`` nor ``end`` is expected to be provided.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ the downloaded resource can be written to.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, but ``end`` is provided, will download from the
+ beginning to ``end`` of the media.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, but ``start`` is provided, will download from the
+ ``start`` to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The response headers must contain
+ a checksum of the requested type. If the headers lack an
+ appropriate checksum (for instance in the case of transcoded or
+ ranged downloads where the remote service does not know the
+ correct checksum) an INFO-level log will be emitted. Supported
+ values are "md5", "crc32c" and None. The default is "md5".
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ """
+
+ async def _write_to_stream(self, response):
+ """Write response body to a write-able stream.
+
+ .. note:
+
+ This method assumes that the ``_stream`` attribute is set on the
+ current download.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the download's
+ checksum doesn't agree with server-computed checksum.
+ """
+
+ # `_get_expected_checksum()` may return None even if a checksum was
+ # requested, in which case it will emit an info log _MISSING_CHECKSUM.
+ # If an invalid checksum type is specified, this will raise ValueError.
+ expected_checksum, checksum_object = sync_helpers._get_expected_checksum(
+ response, self._get_headers, self.media_url, checksum_type=self.checksum
+ )
+
+ async for chunk in response.content.iter_chunked(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE
+ ):
+ self._stream.write(chunk)
+ checksum_object.update(chunk)
+
+ # Don't validate the checksum for partial responses.
+ if (
+ expected_checksum is not None
+ and response.status != http.client.PARTIAL_CONTENT
+ ):
+ actual_checksum = sync_helpers.prepare_checksum_digest(
+ checksum_object.digest()
+ )
+
+ if actual_checksum != expected_checksum:
+ msg = _CHECKSUM_MISMATCH.format(
+ self.media_url,
+ expected_checksum,
+ actual_checksum,
+ checksum_type=self.checksum.upper(),
+ )
+ raise common.DataCorruption(response, msg)
+
+ async def consume(self, transport, timeout=_request_helpers._DEFAULT_TIMEOUT):
+ """Consume the resource to be downloaded.
+
+ If a ``stream`` is attached to this download, then the downloaded
+ resource will be written to the stream.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the download's
+ checksum doesn't agree with server-computed checksum.
+ ValueError: If the current :class:`Download` has already
+ finished.
+ """
+ method, url, payload, headers = self._prepare_request()
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ result = await _request_helpers.http_request(
+ transport,
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ retry_strategy=self._retry_strategy,
+ )
+
+ self._process_response(result)
+
+ if self._stream is not None:
+ await self._write_to_stream(result)
+
+ return result
+
+
+class ChunkedDownload(_request_helpers.RequestsMixin, _download.ChunkedDownload):
+ """Download a resource in chunks from a Google API.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each
+ request.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ will be used to concatenate chunks of the resource as they are
+ downloaded.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, defaults to ``0``.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, will download to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with each request, e.g. headers for data encryption
+ key headers.
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each request.
+
+ Raises:
+ ValueError: If ``start`` is negative.
+ """
+
+ async def consume_next_chunk(
+ self, transport, timeout=_request_helpers._DEFAULT_TIMEOUT
+ ):
+ """
+ Consume the next chunk of the resource to be downloaded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ValueError: If the current download has finished.
+ """
+ method, url, payload, headers = self._prepare_request()
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ result = await _request_helpers.http_request(
+ transport,
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ retry_strategy=self._retry_strategy,
+ timeout=timeout,
+ )
+
+ await self._process_response(result)
+ return result
+
+
+class RawChunkedDownload(_request_helpers.RawRequestsMixin, _download.ChunkedDownload):
+ """Download a raw resource in chunks from a Google API.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each
+ request.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ will be used to concatenate chunks of the resource as they are
+ downloaded.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, defaults to ``0``.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, will download to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with each request, e.g. headers for data encryption
+ key headers.
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each request.
+
+ Raises:
+ ValueError: If ``start`` is negative.
+ """
+
+ async def consume_next_chunk(
+ self, transport, timeout=_request_helpers._DEFAULT_TIMEOUT
+ ):
+ """Consume the next chunk of the resource to be downloaded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ValueError: If the current download has finished.
+ """
+ method, url, payload, headers = self._prepare_request()
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ result = await _request_helpers.http_request(
+ transport,
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ retry_strategy=self._retry_strategy,
+ timeout=timeout,
+ )
+ await self._process_response(result)
+ return result
+
+
+def _add_decoder(response_raw, checksum):
+ """Patch the ``_decoder`` on a ``urllib3`` response.
+
+ This is so that we can intercept the compressed bytes before they are
+ decoded.
+
+ Only patches if the content encoding is ``gzip``.
+
+ Args:
+ response_raw (urllib3.response.HTTPResponse): The raw response for
+ an HTTP request.
+ checksum (object):
+ A checksum which will be updated with compressed bytes.
+
+ Returns:
+ object: Either the original ``checksum`` if ``_decoder`` is not
+ patched, or a ``_DoNothingHash`` if the decoder is patched, since the
+ caller will no longer need to hash to decoded bytes.
+ """
+
+ encoding = response_raw.headers.get("content-encoding", "").lower()
+ if encoding != "gzip":
+ return checksum
+
+ response_raw._decoder = _GzipDecoder(checksum)
+ return _helpers._DoNothingHash()
+
+
+class _GzipDecoder(urllib3.response.GzipDecoder):
+ """Custom subclass of ``urllib3`` decoder for ``gzip``-ed bytes.
+
+ Allows a checksum function to see the compressed bytes before they are
+ decoded. This way the checksum of the compressed value can be computed.
+
+ Args:
+ checksum (object):
+ A checksum which will be updated with compressed bytes.
+ """
+
+ def __init__(self, checksum):
+ super(_GzipDecoder, self).__init__()
+ self._checksum = checksum
+
+ def decompress(self, data, max_length=-1):
+ """Decompress the bytes.
+
+ Args:
+ data (bytes): The compressed bytes to be decompressed.
+ max_length (int): Maximum number of bytes to return. -1 for no
+ limit. Forwarded to the underlying decoder when supported.
+
+ Returns:
+ bytes: The decompressed bytes from ``data``.
+ """
+ self._checksum.update(data)
+ try:
+ return super(_GzipDecoder, self).decompress(data, max_length=max_length)
+ except TypeError:
+ return super(_GzipDecoder, self).decompress(data)
diff --git a/packages/google-resumable-media/google/_async_resumable_media/requests/upload.py b/packages/google-resumable-media/google/_async_resumable_media/requests/upload.py
new file mode 100644
index 000000000000..cbe2f8f02775
--- /dev/null
+++ b/packages/google-resumable-media/google/_async_resumable_media/requests/upload.py
@@ -0,0 +1,514 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Support for resumable uploads.
+
+Also supported here are simple (media) uploads and multipart
+uploads that contain both metadata and a small file as payload.
+"""
+
+from google._async_resumable_media import _upload
+from google._async_resumable_media.requests import _request_helpers
+
+
+class SimpleUpload(_request_helpers.RequestsMixin, _upload.SimpleUpload):
+ """Upload a resource to a Google API.
+
+ A **simple** media upload sends no metadata and completes the upload
+ in a single request.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ async def transmit(
+ self,
+ transport,
+ data,
+ content_type,
+ timeout=_request_helpers._DEFAULT_TIMEOUT,
+ ):
+ """Transmit the resource to be uploaded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ data (bytes): The resource content to be uploaded.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_request(data, content_type)
+
+ response = await _request_helpers.http_request(
+ transport,
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ retry_strategy=self._retry_strategy,
+ timeout=timeout,
+ )
+ self._process_response(response)
+ return response
+
+
+class MultipartUpload(_request_helpers.RequestsMixin, _upload.MultipartUpload):
+ """Upload a resource with metadata to a Google API.
+
+ A **multipart** upload sends both metadata and the resource in a single
+ (multipart) request.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The request metadata will be amended
+ to include the computed value. Using this option will override a
+ manually-set checksum value. Supported values are "md5",
+ "crc32c" and None. The default is None.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ async def transmit(
+ self,
+ transport,
+ data,
+ metadata,
+ content_type,
+ timeout=_request_helpers._DEFAULT_TIMEOUT,
+ ):
+ """Transmit the resource to be uploaded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ data (bytes): The resource content to be uploaded.
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_request(
+ data, metadata, content_type
+ )
+
+ response = await _request_helpers.http_request(
+ transport,
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ retry_strategy=self._retry_strategy,
+ timeout=timeout,
+ )
+ self._process_response(response)
+ return response
+
+
+class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload):
+ """Initiate and fulfill a resumable upload to a Google API.
+
+ A **resumable** upload sends an initial request with the resource metadata
+ and then gets assigned an upload ID / upload URL to send bytes to.
+ Using the upload URL, the upload is then done in chunks (determined by
+ the user) until all bytes have been uploaded.
+
+ When constructing a resumable upload, only the resumable upload URL and
+ the chunk size are required:
+
+ .. testsetup:: resumable-constructor
+
+ bucket = 'bucket-foo'
+
+ .. doctest:: resumable-constructor
+
+ >>> from google.resumable_media.requests import ResumableUpload
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?'
+ ... 'uploadType=resumable')
+ >>> upload_url = url_template.format(bucket=bucket)
+ >>>
+ >>> chunk_size = 3 * 1024 * 1024 # 3MB
+ >>> upload = ResumableUpload(upload_url, chunk_size)
+
+ When initiating an upload (via :meth:`initiate`), the caller is expected
+ to pass the resource being uploaded as a file-like ``stream``. If the size
+ of the resource is explicitly known, it can be passed in directly:
+
+ .. testsetup:: resumable-explicit-size
+
+ import os
+ import tempfile
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import ResumableUpload
+
+ upload_url = 'http://test.invalid'
+ chunk_size = 3 * 1024 * 1024 # 3MB
+ upload = ResumableUpload(upload_url, chunk_size)
+
+ file_desc, filename = tempfile.mkstemp()
+ os.close(file_desc)
+
+ data = b'some bytes!'
+ with open(filename, 'wb') as file_obj:
+ file_obj.write(data)
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response._content = b''
+ resumable_url = 'http://test.invalid?upload_id=7up'
+ fake_response.headers['location'] = resumable_url
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+ .. doctest:: resumable-explicit-size
+
+ >>> import os
+ >>>
+ >>> upload.total_bytes is None
+ True
+ >>>
+ >>> stream = open(filename, 'rb')
+ >>> total_bytes = os.path.getsize(filename)
+ >>> metadata = {'name': filename}
+ >>> response = upload.initiate(
+ ... transport, stream, metadata, 'text/plain',
+ ... total_bytes=total_bytes)
+ >>> response
+
+ >>>
+ >>> upload.total_bytes == total_bytes
+ True
+
+ .. testcleanup:: resumable-explicit-size
+
+ os.remove(filename)
+
+ If the stream is in a "final" state (i.e. it won't have any more bytes
+ written to it), the total number of bytes can be determined implicitly
+ from the ``stream`` itself:
+
+ .. testsetup:: resumable-implicit-size
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import ResumableUpload
+
+ upload_url = 'http://test.invalid'
+ chunk_size = 3 * 1024 * 1024 # 3MB
+ upload = ResumableUpload(upload_url, chunk_size)
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response._content = b''
+ resumable_url = 'http://test.invalid?upload_id=7up'
+ fake_response.headers['location'] = resumable_url
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+ data = b'some MOAR bytes!'
+ metadata = {'name': 'some-file.jpg'}
+ content_type = 'image/jpeg'
+
+ .. doctest:: resumable-implicit-size
+
+ >>> stream = io.BytesIO(data)
+ >>> response = upload.initiate(
+ ... transport, stream, metadata, content_type)
+ >>>
+ >>> upload.total_bytes == len(data)
+ True
+
+ If the size of the resource is **unknown** when the upload is initiated,
+ the ``stream_final`` argument can be used. This might occur if the
+ resource is being dynamically created on the client (e.g. application
+ logs). To use this argument:
+
+ .. testsetup:: resumable-unknown-size
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import ResumableUpload
+
+ upload_url = 'http://test.invalid'
+ chunk_size = 3 * 1024 * 1024 # 3MB
+ upload = ResumableUpload(upload_url, chunk_size)
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response._content = b''
+ resumable_url = 'http://test.invalid?upload_id=7up'
+ fake_response.headers['location'] = resumable_url
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+ metadata = {'name': 'some-file.jpg'}
+ content_type = 'application/octet-stream'
+
+ stream = io.BytesIO(b'data')
+
+ .. doctest:: resumable-unknown-size
+
+ >>> response = upload.initiate(
+ ... transport, stream, metadata, content_type,
+ ... stream_final=False)
+ >>>
+ >>> upload.total_bytes is None
+ True
+
+ Args:
+ upload_url (str): The URL where the resumable upload will be initiated.
+ chunk_size (int): The size of each chunk used to upload the resource.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the :meth:`initiate` request, e.g. headers for
+ encrypted data. These **will not** be sent with
+ :meth:`transmit_next_chunk` or :meth:`recover` requests.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. After the upload is complete, the
+ server-computed checksum of the resulting object will be checked
+ and google.resumable_media.common.DataCorruption will be raised on
+ a mismatch. The corrupted file will not be deleted from the remote
+ host automatically. Supported values are "md5", "crc32c" and None.
+ The default is None.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+
+ Raises:
+ ValueError: If ``chunk_size`` is not a multiple of
+ :data:`.UPLOAD_CHUNK_SIZE`.
+ """
+
+ async def initiate(
+ self,
+ transport,
+ stream,
+ metadata,
+ content_type,
+ total_bytes=None,
+ stream_final=True,
+ timeout=_request_helpers._DEFAULT_TIMEOUT,
+ ):
+ """Initiate a resumable upload.
+
+ By default, this method assumes your ``stream`` is in a "final"
+ state ready to transmit. However, ``stream_final=False`` can be used
+ to indicate that the size of the resource is not known. This can happen
+ if bytes are being dynamically fed into ``stream``, e.g. if the stream
+ is attached to application logs.
+
+ If ``stream_final=False`` is used, :attr:`chunk_size` bytes will be
+ read from the stream every time :meth:`transmit_next_chunk` is called.
+ If one of those reads produces strictly fewer bites than the chunk
+ size, the upload will be concluded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ stream (IO[bytes]): The stream (i.e. file-like object) that will
+ be uploaded. The stream **must** be at the beginning (i.e.
+ ``stream.tell() == 0``).
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ total_bytes (Optional[int]): The total number of bytes to be
+ uploaded. If specified, the upload size **will not** be
+ determined from the stream (even if ``stream_final=True``).
+ stream_final (Optional[bool]): Indicates if the ``stream`` is
+ "final" (i.e. no more bytes will be added to it). In this case
+ we determine the upload size from the size of the stream. If
+ ``total_bytes`` is passed, this argument will be ignored.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_initiate_request(
+ stream,
+ metadata,
+ content_type,
+ total_bytes=total_bytes,
+ stream_final=stream_final,
+ )
+ response = await _request_helpers.http_request(
+ transport,
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ retry_strategy=self._retry_strategy,
+ timeout=timeout,
+ )
+ self._process_initiate_response(response)
+ return response
+
+ async def transmit_next_chunk(
+ self, transport, timeout=_request_helpers._DEFAULT_TIMEOUT
+ ):
+ """Transmit the next chunk of the resource to be uploaded.
+
+ If the current upload was initiated with ``stream_final=False``,
+ this method will dynamically determine if the upload has completed.
+ The upload will be considered complete if the stream produces
+ fewer than :attr:`chunk_size` bytes when a chunk is read from it.
+
+ In the case of failure, an exception is thrown that preserves the
+ failed response:
+
+ .. testsetup:: bad-response
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ from google import resumable_media
+ import google.resumable_media.requests.upload as upload_mod
+
+ transport = mock.Mock(spec=['request'])
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.BAD_REQUEST)
+ transport.request.return_value = fake_response
+
+ upload_url = 'http://test.invalid'
+ upload = upload_mod.ResumableUpload(
+ upload_url, resumable_media.UPLOAD_CHUNK_SIZE)
+ # Fake that the upload has been initiate()-d
+ data = b'data is here'
+ upload._stream = io.BytesIO(data)
+ upload._total_bytes = len(data)
+ upload._resumable_url = 'http://test.invalid?upload_id=nope'
+
+ .. doctest:: bad-response
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> error = None
+ >>> try:
+ ... upload.transmit_next_chunk(transport)
+ ... except resumable_media.InvalidResponse as caught_exc:
+ ... error = caught_exc
+ ...
+ >>> error
+ InvalidResponse('Request failed with status code', 400,
+ 'Expected one of', , 308)
+ >>> error.response
+
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, aiohttp.ClientTimeout]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+ Can also be passed as an `aiohttp.ClientTimeout` object.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200 or 308.
+ ~google.resumable_media.common.DataCorruption: If this is the final
+ chunk, a checksum validation was requested, and the checksum
+ does not match or is not available.
+ """
+ method, url, payload, headers = self._prepare_request()
+ response = await _request_helpers.http_request(
+ transport,
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ retry_strategy=self._retry_strategy,
+ timeout=timeout,
+ )
+ await self._process_resumable_response(response, len(payload))
+ return response
+
+ async def recover(self, transport):
+ """Recover from a failure.
+
+ This method should be used when a :class:`ResumableUpload` is in an
+ :attr:`~ResumableUpload.invalid` state due to a request failure.
+
+ This will verify the progress with the server and make sure the
+ current upload is in a valid state before :meth:`transmit_next_chunk`
+ can be used again.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_recover_request()
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ response = await _request_helpers.http_request(
+ transport,
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ retry_strategy=self._retry_strategy,
+ )
+ self._process_recover_response(response)
+ return response
diff --git a/packages/google-resumable-media/google/resumable_media/__init__.py b/packages/google-resumable-media/google/resumable_media/__init__.py
new file mode 100644
index 000000000000..eaade3e8792f
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/__init__.py
@@ -0,0 +1,60 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for Google Media Downloads and Resumable Uploads.
+
+This package has some general purposes modules, e.g.
+:mod:`~google.resumable_media.common`, but the majority of the
+public interface will be contained in subpackages.
+
+===========
+Subpackages
+===========
+
+Each subpackage is tailored to a specific transport library:
+
+* the :mod:`~google.resumable_media.requests` subpackage uses the ``requests``
+ transport library.
+
+.. _requests: http://docs.python-requests.org/
+
+==========
+Installing
+==========
+
+To install with `pip`_:
+
+.. code-block:: console
+
+ $ pip install --upgrade google-resumable-media
+
+.. _pip: https://pip.pypa.io/
+"""
+
+from google.resumable_media.common import DataCorruption
+from google.resumable_media.common import InvalidResponse
+from google.resumable_media.common import PERMANENT_REDIRECT
+from google.resumable_media.common import RetryStrategy
+from google.resumable_media.common import TOO_MANY_REQUESTS
+from google.resumable_media.common import UPLOAD_CHUNK_SIZE
+
+
+__all__ = [
+ "DataCorruption",
+ "InvalidResponse",
+ "PERMANENT_REDIRECT",
+ "RetryStrategy",
+ "TOO_MANY_REQUESTS",
+ "UPLOAD_CHUNK_SIZE",
+]
diff --git a/packages/google-resumable-media/google/resumable_media/_download.py b/packages/google-resumable-media/google/resumable_media/_download.py
new file mode 100644
index 000000000000..5ca41ba5422e
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/_download.py
@@ -0,0 +1,558 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Virtual bases classes for downloading media from Google APIs."""
+
+import http.client
+import re
+
+from google.resumable_media import _helpers
+from google.resumable_media import common
+
+
+_CONTENT_RANGE_RE = re.compile(
+ r"bytes (?P\d+)-(?P\d+)/(?P\d+)",
+ flags=re.IGNORECASE,
+)
+_ACCEPTABLE_STATUS_CODES = (http.client.OK, http.client.PARTIAL_CONTENT)
+_GET = "GET"
+_ZERO_CONTENT_RANGE_HEADER = "bytes */0"
+
+
+class DownloadBase(object):
+ """Base class for download helpers.
+
+ Defines core shared behavior across different download types.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ the downloaded resource can be written to.
+ start (int): The first byte in a range to be downloaded.
+ end (int): The last byte in a range to be downloaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ """
+
+ def __init__(self, media_url, stream=None, start=None, end=None, headers=None):
+ self.media_url = media_url
+ self._stream = stream
+ self.start = start
+ self.end = end
+ if headers is None:
+ headers = {}
+ self._headers = headers
+ self._finished = False
+ self._retry_strategy = common.RetryStrategy()
+
+ @property
+ def finished(self):
+ """bool: Flag indicating if the download has completed."""
+ return self._finished
+
+ @staticmethod
+ def _get_status_code(response):
+ """Access the status code from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ @staticmethod
+ def _get_headers(response):
+ """Access the headers from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ @staticmethod
+ def _get_body(response):
+ """Access the response body from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class Download(DownloadBase):
+ """Helper to manage downloading a resource from a Google API.
+
+ "Slices" of the resource can be retrieved by specifying a range
+ with ``start`` and / or ``end``. However, in typical usage, neither
+ ``start`` nor ``end`` is expected to be provided.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ the downloaded resource can be written to.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, but ``end`` is provided, will download from the
+ beginning to ``end`` of the media.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, but ``start`` is provided, will download from the
+ ``start`` to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The response headers must contain
+ a checksum of the requested type. If the headers lack an
+ appropriate checksum (for instance in the case of transcoded or
+ ranged downloads where the remote service does not know the
+ correct checksum) an INFO-level log will be emitted. Supported
+ values are "md5", "crc32c" and None.
+ """
+
+ def __init__(
+ self, media_url, stream=None, start=None, end=None, headers=None, checksum="md5"
+ ):
+ super(Download, self).__init__(
+ media_url, stream=stream, start=start, end=end, headers=headers
+ )
+ self.checksum = checksum
+ self._bytes_downloaded = 0
+ self._expected_checksum = None
+ self._checksum_object = None
+ self._object_generation = None
+
+ def _prepare_request(self):
+ """Prepare the contents of an HTTP request.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Returns:
+ Tuple[str, str, NoneType, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always GET)
+ * the URL for the request
+ * the body of the request (always :data:`None`)
+ * headers for the request
+
+ Raises:
+ ValueError: If the current :class:`Download` has already
+ finished.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("A download can only be used once.")
+
+ add_bytes_range(self.start, self.end, self._headers)
+ return _GET, self.media_url, None, self._headers
+
+ def _process_response(self, response):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ # Tombstone the current Download so it cannot be used again.
+ self._finished = True
+ _helpers.require_status_code(
+ response, _ACCEPTABLE_STATUS_CODES, self._get_status_code
+ )
+
+ def consume(self, transport, timeout=None):
+ """Consume the resource to be downloaded.
+
+ If a ``stream`` is attached to this download, then the downloaded
+ resource will be written to the stream.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class ChunkedDownload(DownloadBase):
+ """Download a resource in chunks from a Google API.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each
+ request.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ will be used to concatenate chunks of the resource as they are
+ downloaded.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, defaults to ``0``.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, will download to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with each request, e.g. headers for data encryption
+ key headers.
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each request.
+
+ Raises:
+ ValueError: If ``start`` is negative.
+ """
+
+ def __init__(self, media_url, chunk_size, stream, start=0, end=None, headers=None):
+ if start < 0:
+ raise ValueError(
+ "On a chunked download the starting value cannot be negative."
+ )
+ super(ChunkedDownload, self).__init__(
+ media_url, stream=stream, start=start, end=end, headers=headers
+ )
+ self.chunk_size = chunk_size
+ self._bytes_downloaded = 0
+ self._total_bytes = None
+ self._invalid = False
+
+ @property
+ def bytes_downloaded(self):
+ """int: Number of bytes that have been downloaded."""
+ return self._bytes_downloaded
+
+ @property
+ def total_bytes(self):
+ """Optional[int]: The total number of bytes to be downloaded."""
+ return self._total_bytes
+
+ @property
+ def invalid(self):
+ """bool: Indicates if the download is in an invalid state.
+
+ This will occur if a call to :meth:`consume_next_chunk` fails.
+ """
+ return self._invalid
+
+ def _get_byte_range(self):
+ """Determines the byte range for the next request.
+
+ Returns:
+ Tuple[int, int]: The pair of begin and end byte for the next
+ chunked request.
+ """
+ curr_start = self.start + self.bytes_downloaded
+ curr_end = curr_start + self.chunk_size - 1
+ # Make sure ``curr_end`` does not exceed ``end``.
+ if self.end is not None:
+ curr_end = min(curr_end, self.end)
+ # Make sure ``curr_end`` does not exceed ``total_bytes - 1``.
+ if self.total_bytes is not None:
+ curr_end = min(curr_end, self.total_bytes - 1)
+ return curr_start, curr_end
+
+ def _prepare_request(self):
+ """Prepare the contents of an HTTP request.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ .. note:
+
+ This method will be used multiple times, so ``headers`` will
+ be mutated in between requests. However, we don't make a copy
+ since the same keys are being updated.
+
+ Returns:
+ Tuple[str, str, NoneType, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always GET)
+ * the URL for the request
+ * the body of the request (always :data:`None`)
+ * headers for the request
+
+ Raises:
+ ValueError: If the current download has finished.
+ ValueError: If the current download is invalid.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("Download has finished.")
+ if self.invalid:
+ raise ValueError("Download is invalid and cannot be re-used.")
+
+ curr_start, curr_end = self._get_byte_range()
+ add_bytes_range(curr_start, curr_end, self._headers)
+ return _GET, self.media_url, None, self._headers
+
+ def _make_invalid(self):
+ """Simple setter for ``invalid``.
+
+ This is intended to be passed along as a callback to helpers that
+ raise an exception so they can mark this instance as invalid before
+ raising.
+ """
+ self._invalid = True
+
+ def _process_response(self, response):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O. This is based on the `sans-I/O`_ philosophy.
+
+ For the time being, this **does require** some form of I/O to write
+ a chunk to ``stream``. However, this will (almost) certainly not be
+ network I/O.
+
+ Updates the current state after consuming a chunk. First,
+ increments ``bytes_downloaded`` by the number of bytes in the
+ ``content-length`` header.
+
+ If ``total_bytes`` is already set, this assumes (but does not check)
+ that we already have the correct value and doesn't bother to check
+ that it agrees with the headers.
+
+ We expect the **total** length to be in the ``content-range`` header,
+ but this header is only present on requests which sent the ``range``
+ header. This response header should be of the form
+ ``bytes {start}-{end}/{total}`` and ``{end} - {start} + 1``
+ should be the same as the ``Content-Length``.
+
+ Args:
+ response (object): The HTTP response object (need headers).
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the number
+ of bytes in the body doesn't match the content length header.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ # Verify the response before updating the current instance.
+ if _check_for_zero_content_range(
+ response, self._get_status_code, self._get_headers
+ ):
+ self._finished = True
+ return
+
+ _helpers.require_status_code(
+ response,
+ _ACCEPTABLE_STATUS_CODES,
+ self._get_status_code,
+ callback=self._make_invalid,
+ )
+ headers = self._get_headers(response)
+ response_body = self._get_body(response)
+
+ start_byte, end_byte, total_bytes = get_range_info(
+ response, self._get_headers, callback=self._make_invalid
+ )
+
+ transfer_encoding = headers.get("transfer-encoding")
+
+ if transfer_encoding is None:
+ content_length = _helpers.header_required(
+ response,
+ "content-length",
+ self._get_headers,
+ callback=self._make_invalid,
+ )
+ num_bytes = int(content_length)
+ if len(response_body) != num_bytes:
+ self._make_invalid()
+ raise common.InvalidResponse(
+ response,
+ "Response is different size than content-length",
+ "Expected",
+ num_bytes,
+ "Received",
+ len(response_body),
+ )
+ else:
+ # 'content-length' header not allowed with chunked encoding.
+ num_bytes = end_byte - start_byte + 1
+
+ # First update ``bytes_downloaded``.
+ self._bytes_downloaded += num_bytes
+ # If the end byte is past ``end`` or ``total_bytes - 1`` we are done.
+ if self.end is not None and end_byte >= self.end:
+ self._finished = True
+ elif end_byte >= total_bytes - 1:
+ self._finished = True
+ # NOTE: We only use ``total_bytes`` if not already known.
+ if self.total_bytes is None:
+ self._total_bytes = total_bytes
+ # Write the response body to the stream.
+ self._stream.write(response_body)
+
+ def consume_next_chunk(self, transport, timeout=None):
+ """Consume the next chunk of the resource to be downloaded.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+def add_bytes_range(start, end, headers):
+ """Add a bytes range to a header dictionary.
+
+ Some possible inputs and the corresponding bytes ranges::
+
+ >>> headers = {}
+ >>> add_bytes_range(None, None, headers)
+ >>> headers
+ {}
+ >>> add_bytes_range(500, 999, headers)
+ >>> headers['range']
+ 'bytes=500-999'
+ >>> add_bytes_range(None, 499, headers)
+ >>> headers['range']
+ 'bytes=0-499'
+ >>> add_bytes_range(-500, None, headers)
+ >>> headers['range']
+ 'bytes=-500'
+ >>> add_bytes_range(9500, None, headers)
+ >>> headers['range']
+ 'bytes=9500-'
+
+ Args:
+ start (Optional[int]): The first byte in a range. Can be zero,
+ positive, negative or :data:`None`.
+ end (Optional[int]): The last byte in a range. Assumed to be
+ positive.
+ headers (Mapping[str, str]): A headers mapping which can have the
+ bytes range added if at least one of ``start`` or ``end``
+ is not :data:`None`.
+ """
+ if start is None:
+ if end is None:
+ # No range to add.
+ return
+ else:
+ # NOTE: This assumes ``end`` is non-negative.
+ bytes_range = "0-{:d}".format(end)
+ else:
+ if end is None:
+ if start < 0:
+ bytes_range = "{:d}".format(start)
+ else:
+ bytes_range = "{:d}-".format(start)
+ else:
+ # NOTE: This is invalid if ``start < 0``.
+ bytes_range = "{:d}-{:d}".format(start, end)
+
+ headers[_helpers.RANGE_HEADER] = "bytes=" + bytes_range
+
+
+def get_range_info(response, get_headers, callback=_helpers.do_nothing):
+ """Get the start, end and total bytes from a content range header.
+
+ Args:
+ response (object): An HTTP response object.
+ get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
+ from an HTTP response.
+ callback (Optional[Callable]): A callback that takes no arguments,
+ to be executed when an exception is being raised.
+
+ Returns:
+ Tuple[int, int, int]: The start byte, end byte and total bytes.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the
+ ``Content-Range`` header is not of the form
+ ``bytes {start}-{end}/{total}``.
+ """
+ content_range = _helpers.header_required(
+ response, _helpers.CONTENT_RANGE_HEADER, get_headers, callback=callback
+ )
+ match = _CONTENT_RANGE_RE.match(content_range)
+ if match is None:
+ callback()
+ raise common.InvalidResponse(
+ response,
+ "Unexpected content-range header",
+ content_range,
+ 'Expected to be of the form "bytes {start}-{end}/{total}"',
+ )
+
+ return (
+ int(match.group("start_byte")),
+ int(match.group("end_byte")),
+ int(match.group("total_bytes")),
+ )
+
+
+def _check_for_zero_content_range(response, get_status_code, get_headers):
+ """Validate if response status code is 416 and content range is zero.
+
+ This is the special case for handling zero bytes files.
+
+ Args:
+ response (object): An HTTP response object.
+ get_status_code (Callable[Any, int]): Helper to get a status code
+ from a response.
+ get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
+ from an HTTP response.
+
+ Returns:
+ bool: True if content range total bytes is zero, false otherwise.
+ """
+ if get_status_code(response) == http.client.REQUESTED_RANGE_NOT_SATISFIABLE:
+ content_range = _helpers.header_required(
+ response,
+ _helpers.CONTENT_RANGE_HEADER,
+ get_headers,
+ callback=_helpers.do_nothing,
+ )
+ if content_range == _ZERO_CONTENT_RANGE_HEADER:
+ return True
+ return False
diff --git a/packages/google-resumable-media/google/resumable_media/_helpers.py b/packages/google-resumable-media/google/resumable_media/_helpers.py
new file mode 100644
index 000000000000..e80d9909b6bf
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/_helpers.py
@@ -0,0 +1,434 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared utilities used by both downloads and uploads."""
+
+from __future__ import absolute_import
+
+import base64
+import hashlib
+import logging
+import random
+import warnings
+
+from urllib.parse import parse_qs
+from urllib.parse import urlencode
+from urllib.parse import urlsplit
+from urllib.parse import urlunsplit
+
+from google.resumable_media import common
+
+
+RANGE_HEADER = "range"
+CONTENT_RANGE_HEADER = "content-range"
+CONTENT_ENCODING_HEADER = "content-encoding"
+
+_SLOW_CRC32C_WARNING = (
+ "Currently using crcmod in pure python form. This is a slow "
+ "implementation. Python 3 has a faster implementation, `google-crc32c`, "
+ "which will be used if it is installed."
+)
+_GENERATION_HEADER = "x-goog-generation"
+_HASH_HEADER = "x-goog-hash"
+_STORED_CONTENT_ENCODING_HEADER = "x-goog-stored-content-encoding"
+
+_MISSING_CHECKSUM = """\
+No {checksum_type} checksum was returned from the service while downloading {}
+(which happens for composite objects), so client-side content integrity
+checking is not being performed."""
+_LOGGER = logging.getLogger(__name__)
+
+
+def do_nothing():
+ """Simple default callback."""
+
+
+def header_required(response, name, get_headers, callback=do_nothing):
+ """Checks that a specific header is in a headers dictionary.
+
+ Args:
+ response (object): An HTTP response object, expected to have a
+ ``headers`` attribute that is a ``Mapping[str, str]``.
+ name (str): The name of a required header.
+ get_headers (Callable[Any, Mapping[str, str]]): Helper to get headers
+ from an HTTP response.
+ callback (Optional[Callable]): A callback that takes no arguments,
+ to be executed when an exception is being raised.
+
+ Returns:
+ str: The desired header.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the header
+ is missing.
+ """
+ headers = get_headers(response)
+ if name not in headers:
+ callback()
+ raise common.InvalidResponse(
+ response, "Response headers must contain header", name
+ )
+
+ return headers[name]
+
+
+def require_status_code(response, status_codes, get_status_code, callback=do_nothing):
+ """Require a response has a status code among a list.
+
+ Args:
+ response (object): The HTTP response object.
+ status_codes (tuple): The acceptable status codes.
+ get_status_code (Callable[Any, int]): Helper to get a status code
+ from a response.
+ callback (Optional[Callable]): A callback that takes no arguments,
+ to be executed when an exception is being raised.
+
+ Returns:
+ int: The status code.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status code
+ is not one of the values in ``status_codes``.
+ """
+ status_code = get_status_code(response)
+ if status_code not in status_codes:
+ if status_code not in common.RETRYABLE:
+ callback()
+ raise common.InvalidResponse(
+ response,
+ "Request failed with status code",
+ status_code,
+ "Expected one of",
+ *status_codes,
+ )
+ return status_code
+
+
+def calculate_retry_wait(base_wait, max_sleep, multiplier=2.0):
+ """Calculate the amount of time to wait before a retry attempt.
+
+ Wait time grows exponentially with the number of attempts, until
+ ``max_sleep``.
+
+ A random amount of jitter (between 0 and 1 seconds) is added to spread out
+ retry attempts from different clients.
+
+ Args:
+ base_wait (float): The "base" wait time (i.e. without any jitter)
+ that will be multiplied until it reaches the maximum sleep.
+ max_sleep (float): Maximum value that a sleep time is allowed to be.
+ multiplier (float): Multiplier to apply to the base wait.
+
+ Returns:
+ Tuple[float, float]: The new base wait time as well as the wait time
+ to be applied (with a random amount of jitter between 0 and 1 seconds
+ added).
+ """
+ new_base_wait = multiplier * base_wait
+ if new_base_wait > max_sleep:
+ new_base_wait = max_sleep
+
+ jitter_ms = random.randint(0, 1000)
+ return new_base_wait, new_base_wait + 0.001 * jitter_ms
+
+
+def _get_crc32c_object():
+ """Get crc32c object
+ Attempt to use the Google-CRC32c package. If it isn't available, try
+ to use CRCMod. CRCMod might be using a 'slow' varietal. If so, warn...
+ """
+ try:
+ import google_crc32c # type: ignore
+
+ crc_obj = google_crc32c.Checksum()
+ except ImportError:
+ try:
+ import crcmod # type: ignore
+
+ crc_obj = crcmod.predefined.Crc("crc-32c")
+ _is_fast_crcmod()
+
+ except ImportError:
+ raise ImportError("Failed to import either `google-crc32c` or `crcmod`")
+
+ return crc_obj
+
+
+def _is_fast_crcmod():
+ # Determine if this is using the slow form of crcmod.
+ nested_crcmod = __import__(
+ "crcmod.crcmod",
+ globals(),
+ locals(),
+ ["_usingExtension"],
+ 0,
+ )
+ fast_crc = getattr(nested_crcmod, "_usingExtension", False)
+ if not fast_crc:
+ warnings.warn(_SLOW_CRC32C_WARNING, RuntimeWarning, stacklevel=2)
+ return fast_crc
+
+
+def _get_metadata_key(checksum_type):
+ if checksum_type == "md5":
+ return "md5Hash"
+ else:
+ return checksum_type
+
+
+def prepare_checksum_digest(digest_bytestring):
+ """Convert a checksum object into a digest encoded for an HTTP header.
+
+ Args:
+ bytes: A checksum digest bytestring.
+
+ Returns:
+ str: A base64 string representation of the input.
+ """
+ encoded_digest = base64.b64encode(digest_bytestring)
+ # NOTE: ``b64encode`` returns ``bytes``, but HTTP headers expect ``str``.
+ return encoded_digest.decode("utf-8")
+
+
+def _get_expected_checksum(response, get_headers, media_url, checksum_type):
+ """Get the expected checksum and checksum object for the download response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+ get_headers (callable: response->dict): returns response headers.
+ media_url (str): The URL containing the media to be downloaded.
+ checksum_type Optional(str): The checksum type to read from the headers,
+ exactly as it will appear in the headers (case-sensitive). Must be
+ "md5", "crc32c" or None.
+
+ Returns:
+ Tuple (Optional[str], object): The expected checksum of the response,
+ if it can be detected from the ``X-Goog-Hash`` header, and the
+ appropriate checksum object for the expected checksum.
+ """
+ if checksum_type not in ["md5", "crc32c", None]:
+ raise ValueError("checksum must be ``'md5'``, ``'crc32c'`` or ``None``")
+ elif checksum_type in ["md5", "crc32c"]:
+ headers = get_headers(response)
+ expected_checksum = _parse_checksum_header(
+ headers.get(_HASH_HEADER), response, checksum_label=checksum_type
+ )
+
+ if expected_checksum is None:
+ msg = _MISSING_CHECKSUM.format(
+ media_url, checksum_type=checksum_type.upper()
+ )
+ _LOGGER.info(msg)
+ checksum_object = _DoNothingHash()
+ else:
+ if checksum_type == "md5":
+ checksum_object = hashlib.md5()
+ else:
+ checksum_object = _get_crc32c_object()
+ else:
+ expected_checksum = None
+ checksum_object = _DoNothingHash()
+
+ return (expected_checksum, checksum_object)
+
+
+def _get_uploaded_checksum_from_headers(response, get_headers, checksum_type):
+ """Get the computed checksum and checksum object from the response headers.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+ get_headers (callable: response->dict): returns response headers.
+ checksum_type Optional(str): The checksum type to read from the headers,
+ exactly as it will appear in the headers (case-sensitive). Must be
+ "md5", "crc32c" or None.
+
+ Returns:
+ Tuple (Optional[str], object): The checksum of the response,
+ if it can be detected from the ``X-Goog-Hash`` header, and the
+ appropriate checksum object for the expected checksum.
+ """
+ if checksum_type not in ["md5", "crc32c", None]:
+ raise ValueError("checksum must be ``'md5'``, ``'crc32c'`` or ``None``")
+ elif checksum_type in ["md5", "crc32c"]:
+ headers = get_headers(response)
+ remote_checksum = _parse_checksum_header(
+ headers.get(_HASH_HEADER), response, checksum_label=checksum_type
+ )
+ else:
+ remote_checksum = None
+
+ return remote_checksum
+
+
+def _parse_checksum_header(header_value, response, checksum_label):
+ """Parses the checksum header from an ``X-Goog-Hash`` value.
+
+ .. _header reference: https://cloud.google.com/storage/docs/\
+ xml-api/reference-headers#xgooghash
+
+ Expects ``header_value`` (if not :data:`None`) to be in one of the three
+ following formats:
+
+ * ``crc32c=n03x6A==``
+ * ``md5=Ojk9c3dhfxgoKVVHYwFbHQ==``
+ * ``crc32c=n03x6A==,md5=Ojk9c3dhfxgoKVVHYwFbHQ==``
+
+ See the `header reference`_ for more information.
+
+ Args:
+ header_value (Optional[str]): The ``X-Goog-Hash`` header from
+ a download response.
+ response (~requests.Response): The HTTP response object.
+ checksum_label (str): The label of the header value to read, as in the
+ examples above. Typically "md5" or "crc32c"
+
+ Returns:
+ Optional[str]: The expected checksum of the response, if it
+ can be detected from the ``X-Goog-Hash`` header; otherwise, None.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If there are
+ multiple checksums of the requested type in ``header_value``.
+ """
+ if header_value is None:
+ return None
+
+ matches = []
+ for checksum in header_value.split(","):
+ name, value = checksum.split("=", 1)
+ # Official docs say "," is the separator, but real-world responses have encountered ", "
+ if name.lstrip() == checksum_label:
+ matches.append(value)
+
+ if len(matches) == 0:
+ return None
+ elif len(matches) == 1:
+ return matches[0]
+ else:
+ raise common.InvalidResponse(
+ response,
+ "X-Goog-Hash header had multiple ``{}`` values.".format(checksum_label),
+ header_value,
+ matches,
+ )
+
+
+def _get_checksum_object(checksum_type):
+ """Respond with a checksum object for a supported type, if not None.
+
+ Raises ValueError if checksum_type is unsupported.
+ """
+ if checksum_type == "md5":
+ return hashlib.md5()
+ elif checksum_type == "crc32c":
+ return _get_crc32c_object()
+ elif checksum_type is None:
+ return None
+ else:
+ raise ValueError("checksum must be ``'md5'``, ``'crc32c'`` or ``None``")
+
+
+def _parse_generation_header(response, get_headers):
+ """Parses the generation header from an ``X-Goog-Generation`` value.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+ get_headers (callable: response->dict): returns response headers.
+
+ Returns:
+ Optional[long]: The object generation from the response, if it
+ can be detected from the ``X-Goog-Generation`` header; otherwise, None.
+ """
+ headers = get_headers(response)
+ object_generation = headers.get(_GENERATION_HEADER, None)
+
+ if object_generation is None:
+ return None
+ else:
+ return int(object_generation)
+
+
+def _get_generation_from_url(media_url):
+ """Retrieve the object generation query param specified in the media url.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+
+ Returns:
+ long: The object generation from the media url if exists; otherwise, None.
+ """
+
+ _, _, _, query, _ = urlsplit(media_url)
+ query_params = parse_qs(query)
+ object_generation = query_params.get("generation", None)
+
+ if object_generation is None:
+ return None
+ else:
+ return int(object_generation[0])
+
+
+def add_query_parameters(media_url, query_params):
+ """Add query parameters to a base url.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ query_params (dict): Names and values of the query parameters to add.
+
+ Returns:
+ str: URL with additional query strings appended.
+ """
+
+ if len(query_params) == 0:
+ return media_url
+
+ scheme, netloc, path, query, frag = urlsplit(media_url)
+ params = parse_qs(query)
+ new_params = {**params, **query_params}
+ query = urlencode(new_params, doseq=True)
+ return urlunsplit((scheme, netloc, path, query, frag))
+
+
+def _is_decompressive_transcoding(response, get_headers):
+ """Returns True if the object was served decompressed. This happens when the
+ "x-goog-stored-content-encoding" header is "gzip" and "content-encoding" header
+ is not "gzip". See more at: https://cloud.google.com/storage/docs/transcoding#transcoding_and_gzip
+ Args:
+ response (~requests.Response): The HTTP response object.
+ get_headers (callable: response->dict): returns response headers.
+ Returns:
+ bool: Returns True if decompressive transcoding has occurred; otherwise, False.
+ """
+ headers = get_headers(response)
+ return (
+ headers.get(_STORED_CONTENT_ENCODING_HEADER) == "gzip"
+ and headers.get(CONTENT_ENCODING_HEADER) != "gzip"
+ )
+
+
+class _DoNothingHash(object):
+ """Do-nothing hash object.
+
+ Intended as a stand-in for ``hashlib.md5`` or a crc32c checksum
+ implementation in cases where it isn't necessary to compute the hash.
+ """
+
+ def update(self, unused_chunk):
+ """Do-nothing ``update`` method.
+
+ Intended to match the interface of ``hashlib.md5`` and other checksums.
+
+ Args:
+ unused_chunk (bytes): A chunk of data.
+ """
diff --git a/packages/google-resumable-media/google/resumable_media/_upload.py b/packages/google-resumable-media/google/resumable_media/_upload.py
new file mode 100644
index 000000000000..e176fb391d8c
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/_upload.py
@@ -0,0 +1,1531 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Virtual bases classes for uploading media via Google APIs.
+
+Supported here are:
+
+* simple (media) uploads
+* multipart uploads that contain both metadata and a small file as payload
+* resumable uploads (with metadata as well)
+"""
+
+import http.client
+import json
+import os
+import random
+import re
+import sys
+import urllib.parse
+
+from google import resumable_media
+from google.resumable_media import _helpers
+from google.resumable_media import common
+
+from xml.etree import ElementTree
+
+
+_CONTENT_TYPE_HEADER = "content-type"
+_CONTENT_RANGE_TEMPLATE = "bytes {:d}-{:d}/{:d}"
+_RANGE_UNKNOWN_TEMPLATE = "bytes {:d}-{:d}/*"
+_EMPTY_RANGE_TEMPLATE = "bytes */{:d}"
+_BOUNDARY_WIDTH = len(str(sys.maxsize - 1))
+_BOUNDARY_FORMAT = "==============={{:0{:d}d}}==".format(_BOUNDARY_WIDTH)
+_MULTIPART_SEP = b"--"
+_CRLF = b"\r\n"
+_MULTIPART_BEGIN = b"\r\ncontent-type: application/json; charset=UTF-8\r\n\r\n"
+_RELATED_HEADER = b'multipart/related; boundary="'
+_BYTES_RANGE_RE = re.compile(r"bytes=0-(?P\d+)", flags=re.IGNORECASE)
+_STREAM_ERROR_TEMPLATE = (
+ "Bytes stream is in unexpected state. "
+ "The local stream has had {:d} bytes read from it while "
+ "{:d} bytes have already been updated (they should match)."
+)
+_STREAM_READ_PAST_TEMPLATE = (
+ "{:d} bytes have been read from the stream, which exceeds the expected total {:d}."
+)
+_DELETE = "DELETE"
+_POST = "POST"
+_PUT = "PUT"
+_UPLOAD_CHECKSUM_MISMATCH_MESSAGE = (
+ "The computed ``{}`` checksum, ``{}``, and the checksum reported by the "
+ "remote host, ``{}``, did not match."
+)
+_UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE = (
+ "Response metadata had no ``{}`` value; checksum could not be validated."
+)
+_UPLOAD_HEADER_NO_APPROPRIATE_CHECKSUM_MESSAGE = (
+ "Response headers had no ``{}`` value; checksum could not be validated."
+)
+_MPU_INITIATE_QUERY = "?uploads"
+_MPU_PART_QUERY_TEMPLATE = "?partNumber={part}&uploadId={upload_id}"
+_S3_COMPAT_XML_NAMESPACE = "{http://s3.amazonaws.com/doc/2006-03-01/}"
+_UPLOAD_ID_NODE = "UploadId"
+_MPU_FINAL_QUERY_TEMPLATE = "?uploadId={upload_id}"
+
+
+class UploadBase(object):
+ """Base class for upload helpers.
+
+ Defines core shared behavior across different upload types.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ def __init__(self, upload_url, headers=None):
+ self.upload_url = upload_url
+ if headers is None:
+ headers = {}
+ self._headers = headers
+ self._finished = False
+ self._retry_strategy = common.RetryStrategy()
+
+ @property
+ def finished(self):
+ """bool: Flag indicating if the upload has completed."""
+ return self._finished
+
+ def _process_response(self, response):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ # Tombstone the current upload so it cannot be used again (in either
+ # failure or success).
+ self._finished = True
+ _helpers.require_status_code(response, (http.client.OK,), self._get_status_code)
+
+ @staticmethod
+ def _get_status_code(response):
+ """Access the status code from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ @staticmethod
+ def _get_headers(response):
+ """Access the headers from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ @staticmethod
+ def _get_body(response):
+ """Access the response body from an HTTP response.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class SimpleUpload(UploadBase):
+ """Upload a resource to a Google API.
+
+ A **simple** media upload sends no metadata and completes the upload
+ in a single request.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ def _prepare_request(self, data, content_type):
+ """Prepare the contents of an HTTP request.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ .. note:
+
+ This method will be used only once, so ``headers`` will be
+ mutated by having a new key added to it.
+
+ Args:
+ data (bytes): The resource content to be uploaded.
+ content_type (str): The content type for the request.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the current upload has already finished.
+ TypeError: If ``data`` isn't bytes.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("An upload can only be used once.")
+
+ if not isinstance(data, bytes):
+ raise TypeError("`data` must be bytes, received", type(data))
+ self._headers[_CONTENT_TYPE_HEADER] = content_type
+ return _POST, self.upload_url, data, self._headers
+
+ def transmit(self, transport, data, content_type, timeout=None):
+ """Transmit the resource to be uploaded.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ data (bytes): The resource content to be uploaded.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class MultipartUpload(UploadBase):
+ """Upload a resource with metadata to a Google API.
+
+ A **multipart** upload sends both metadata and the resource in a single
+ (multipart) request.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum (Optional([str])): The type of checksum to compute to verify
+ the integrity of the object. The request metadata will be amended
+ to include the computed value. Using this option will override a
+ manually-set checksum value. Supported values are "md5", "crc32c"
+ and None. The default is None.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ def __init__(self, upload_url, headers=None, checksum=None):
+ super(MultipartUpload, self).__init__(upload_url, headers=headers)
+ self._checksum_type = checksum
+
+ def _prepare_request(self, data, metadata, content_type):
+ """Prepare the contents of an HTTP request.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ .. note:
+
+ This method will be used only once, so ``headers`` will be
+ mutated by having a new key added to it.
+
+ Args:
+ data (bytes): The resource content to be uploaded.
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the current upload has already finished.
+ TypeError: If ``data`` isn't bytes.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("An upload can only be used once.")
+
+ if not isinstance(data, bytes):
+ raise TypeError("`data` must be bytes, received", type(data))
+
+ checksum_object = _helpers._get_checksum_object(self._checksum_type)
+ if checksum_object is not None:
+ checksum_object.update(data)
+ actual_checksum = _helpers.prepare_checksum_digest(checksum_object.digest())
+ metadata_key = _helpers._get_metadata_key(self._checksum_type)
+ metadata[metadata_key] = actual_checksum
+
+ content, multipart_boundary = construct_multipart_request(
+ data, metadata, content_type
+ )
+ multipart_content_type = _RELATED_HEADER + multipart_boundary + b'"'
+ self._headers[_CONTENT_TYPE_HEADER] = multipart_content_type
+
+ return _POST, self.upload_url, content, self._headers
+
+ def transmit(self, transport, data, metadata, content_type, timeout=None):
+ """Transmit the resource to be uploaded.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ data (bytes): The resource content to be uploaded.
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class ResumableUpload(UploadBase):
+ """Initiate and fulfill a resumable upload to a Google API.
+
+ A **resumable** upload sends an initial request with the resource metadata
+ and then gets assigned an upload ID / upload URL to send bytes to.
+ Using the upload URL, the upload is then done in chunks (determined by
+ the user) until all bytes have been uploaded.
+
+ Args:
+ upload_url (str): The URL where the resumable upload will be initiated.
+ chunk_size (int): The size of each chunk used to upload the resource.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with every request.
+ checksum (Optional([str])): The type of checksum to compute to verify
+ the integrity of the object. After the upload is complete, the
+ server-computed checksum of the resulting object will be read
+ and google.resumable_media.common.DataCorruption will be raised on
+ a mismatch. The corrupted file will not be deleted from the remote
+ host automatically. Supported values are "md5", "crc32c" and None.
+ The default is None.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+
+ Raises:
+ ValueError: If ``chunk_size`` is not a multiple of
+ :data:`.UPLOAD_CHUNK_SIZE`.
+ """
+
+ def __init__(self, upload_url, chunk_size, checksum=None, headers=None):
+ super(ResumableUpload, self).__init__(upload_url, headers=headers)
+ if chunk_size % resumable_media.UPLOAD_CHUNK_SIZE != 0:
+ raise ValueError(
+ "{} KB must divide chunk size".format(
+ resumable_media.UPLOAD_CHUNK_SIZE / 1024
+ )
+ )
+ self._chunk_size = chunk_size
+ self._stream = None
+ self._content_type = None
+ self._bytes_uploaded = 0
+ self._bytes_checksummed = 0
+ self._checksum_type = checksum
+ self._checksum_object = None
+ self._total_bytes = None
+ self._resumable_url = None
+ self._invalid = False
+
+ @property
+ def invalid(self):
+ """bool: Indicates if the upload is in an invalid state.
+
+ This will occur if a call to :meth:`transmit_next_chunk` fails.
+ To recover from such a failure, call :meth:`recover`.
+ """
+ return self._invalid
+
+ @property
+ def chunk_size(self):
+ """int: The size of each chunk used to upload the resource."""
+ return self._chunk_size
+
+ @property
+ def resumable_url(self):
+ """Optional[str]: The URL of the in-progress resumable upload."""
+ return self._resumable_url
+
+ @property
+ def bytes_uploaded(self):
+ """int: Number of bytes that have been uploaded."""
+ return self._bytes_uploaded
+
+ @property
+ def total_bytes(self):
+ """Optional[int]: The total number of bytes to be uploaded.
+
+ If this upload is initiated (via :meth:`initiate`) with
+ ``stream_final=True``, this value will be populated based on the size
+ of the ``stream`` being uploaded. (By default ``stream_final=True``.)
+
+ If this upload is initiated with ``stream_final=False``,
+ :attr:`total_bytes` will be :data:`None` since it cannot be
+ determined from the stream.
+ """
+ return self._total_bytes
+
+ def _prepare_initiate_request(
+ self, stream, metadata, content_type, total_bytes=None, stream_final=True
+ ):
+ """Prepare the contents of HTTP request to initiate upload.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ stream (IO[bytes]): The stream (i.e. file-like object) that will
+ be uploaded. The stream **must** be at the beginning (i.e.
+ ``stream.tell() == 0``).
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ total_bytes (Optional[int]): The total number of bytes to be
+ uploaded. If specified, the upload size **will not** be
+ determined from the stream (even if ``stream_final=True``).
+ stream_final (Optional[bool]): Indicates if the ``stream`` is
+ "final" (i.e. no more bytes will be added to it). In this case
+ we determine the upload size from the size of the stream. If
+ ``total_bytes`` is passed, this argument will be ignored.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the current upload has already been initiated.
+ ValueError: If ``stream`` is not at the beginning.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.resumable_url is not None:
+ raise ValueError("This upload has already been initiated.")
+ if stream.tell() != 0:
+ raise ValueError("Stream must be at beginning.")
+
+ self._stream = stream
+ self._content_type = content_type
+
+ # Signed URL requires content type set directly - not through x-upload-content-type
+ parse_result = urllib.parse.urlparse(self.upload_url)
+ parsed_query = urllib.parse.parse_qs(parse_result.query)
+ if "x-goog-signature" in parsed_query or "X-Goog-Signature" in parsed_query:
+ # Deconstruct **self._headers first so that content type defined here takes priority
+ headers = {**self._headers, _CONTENT_TYPE_HEADER: content_type}
+ else:
+ # Deconstruct **self._headers first so that content type defined here takes priority
+ headers = {
+ **self._headers,
+ _CONTENT_TYPE_HEADER: "application/json; charset=UTF-8",
+ "x-upload-content-type": content_type,
+ }
+ # Set the total bytes if possible.
+ if total_bytes is not None:
+ self._total_bytes = total_bytes
+ elif stream_final:
+ self._total_bytes = get_total_bytes(stream)
+ # Add the total bytes to the headers if set.
+ if self._total_bytes is not None:
+ content_length = "{:d}".format(self._total_bytes)
+ headers["x-upload-content-length"] = content_length
+
+ payload = json.dumps(metadata).encode("utf-8")
+ return _POST, self.upload_url, payload, headers
+
+ def _process_initiate_response(self, response):
+ """Process the response from an HTTP request that initiated upload.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ This method takes the URL from the ``Location`` header and stores it
+ for future use. Within that URL, we assume the ``upload_id`` query
+ parameter has been included, but we do not check.
+
+ Args:
+ response (object): The HTTP response object (need headers).
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ _helpers.require_status_code(
+ response,
+ (http.client.OK, http.client.CREATED),
+ self._get_status_code,
+ callback=self._make_invalid,
+ )
+ self._resumable_url = _helpers.header_required(
+ response, "location", self._get_headers
+ )
+
+ def initiate(
+ self,
+ transport,
+ stream,
+ metadata,
+ content_type,
+ total_bytes=None,
+ stream_final=True,
+ timeout=None,
+ ):
+ """Initiate a resumable upload.
+
+ By default, this method assumes your ``stream`` is in a "final"
+ state ready to transmit. However, ``stream_final=False`` can be used
+ to indicate that the size of the resource is not known. This can happen
+ if bytes are being dynamically fed into ``stream``, e.g. if the stream
+ is attached to application logs.
+
+ If ``stream_final=False`` is used, :attr:`chunk_size` bytes will be
+ read from the stream every time :meth:`transmit_next_chunk` is called.
+ If one of those reads produces strictly fewer bites than the chunk
+ size, the upload will be concluded.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ stream (IO[bytes]): The stream (i.e. file-like object) that will
+ be uploaded. The stream **must** be at the beginning (i.e.
+ ``stream.tell() == 0``).
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ total_bytes (Optional[int]): The total number of bytes to be
+ uploaded. If specified, the upload size **will not** be
+ determined from the stream (even if ``stream_final=True``).
+ stream_final (Optional[bool]): Indicates if the ``stream`` is
+ "final" (i.e. no more bytes will be added to it). In this case
+ we determine the upload size from the size of the stream. If
+ ``total_bytes`` is passed, this argument will be ignored.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _prepare_request(self):
+ """Prepare the contents of HTTP request to upload a chunk.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O. This is based on the `sans-I/O`_ philosophy.
+
+ For the time being, this **does require** some form of I/O to read
+ a chunk from ``stream`` (via :func:`get_next_chunk`). However, this
+ will (almost) certainly not be network I/O.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always PUT)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ The headers incorporate the ``_headers`` on the current instance.
+
+ Raises:
+ ValueError: If the current upload has finished.
+ ValueError: If the current upload is in an invalid state.
+ ValueError: If the current upload has not been initiated.
+ ValueError: If the location in the stream (i.e. ``stream.tell()``)
+ does not agree with ``bytes_uploaded``.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("Upload has finished.")
+ if self.invalid:
+ raise ValueError(
+ "Upload is in an invalid state. To recover call `recover()`."
+ )
+ if self.resumable_url is None:
+ raise ValueError(
+ "This upload has not been initiated. Please call "
+ "initiate() before beginning to transmit chunks."
+ )
+
+ start_byte, payload, content_range = get_next_chunk(
+ self._stream, self._chunk_size, self._total_bytes
+ )
+ if start_byte != self.bytes_uploaded:
+ msg = _STREAM_ERROR_TEMPLATE.format(start_byte, self.bytes_uploaded)
+ raise ValueError(msg)
+
+ self._update_checksum(start_byte, payload)
+
+ headers = {
+ **self._headers,
+ _CONTENT_TYPE_HEADER: self._content_type,
+ _helpers.CONTENT_RANGE_HEADER: content_range,
+ }
+ return _PUT, self.resumable_url, payload, headers
+
+ def _update_checksum(self, start_byte, payload):
+ """Update the checksum with the payload if not already updated.
+
+ Because error recovery can result in bytes being transmitted more than
+ once, the checksum tracks the number of bytes checked in
+ self._bytes_checksummed and skips bytes that have already been summed.
+ """
+ if not self._checksum_type:
+ return
+
+ if not self._checksum_object:
+ self._checksum_object = _helpers._get_checksum_object(self._checksum_type)
+
+ if start_byte < self._bytes_checksummed:
+ offset = self._bytes_checksummed - start_byte
+ data = payload[offset:]
+ else:
+ data = payload
+
+ self._checksum_object.update(data)
+ self._bytes_checksummed += len(data)
+
+ def _make_invalid(self):
+ """Simple setter for ``invalid``.
+
+ This is intended to be passed along as a callback to helpers that
+ raise an exception so they can mark this instance as invalid before
+ raising.
+ """
+ self._invalid = True
+
+ def _process_resumable_response(self, response, bytes_sent):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+ bytes_sent (int): The number of bytes sent in the request that
+ ``response`` was returned for.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is 308 and the ``range`` header is not of the form
+ ``bytes 0-{end}``.
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200 or 308.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ status_code = _helpers.require_status_code(
+ response,
+ (http.client.OK, http.client.PERMANENT_REDIRECT),
+ self._get_status_code,
+ callback=self._make_invalid,
+ )
+ if status_code == http.client.OK:
+ # NOTE: We use the "local" information of ``bytes_sent`` to update
+ # ``bytes_uploaded``, but do not verify this against other
+ # state. However, there may be some other information:
+ #
+ # * a ``size`` key in JSON response body
+ # * the ``total_bytes`` attribute (if set)
+ # * ``stream.tell()`` (relying on fact that ``initiate()``
+ # requires stream to be at the beginning)
+ self._bytes_uploaded = self._bytes_uploaded + bytes_sent
+ # Tombstone the current upload so it cannot be used again.
+ self._finished = True
+ # Validate the checksum. This can raise an exception on failure.
+ self._validate_checksum(response)
+ else:
+ bytes_range = _helpers.header_required(
+ response,
+ _helpers.RANGE_HEADER,
+ self._get_headers,
+ callback=self._make_invalid,
+ )
+ match = _BYTES_RANGE_RE.match(bytes_range)
+ if match is None:
+ self._make_invalid()
+ raise common.InvalidResponse(
+ response,
+ 'Unexpected "range" header',
+ bytes_range,
+ 'Expected to be of the form "bytes=0-{end}"',
+ )
+ self._bytes_uploaded = int(match.group("end_byte")) + 1
+
+ def _validate_checksum(self, response):
+ """Check the computed checksum, if any, against the recieved metadata.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the checksum
+ computed locally and the checksum reported by the remote host do
+ not match.
+ """
+ if self._checksum_type is None:
+ return
+ metadata_key = _helpers._get_metadata_key(self._checksum_type)
+ metadata = response.json()
+ remote_checksum = metadata.get(metadata_key)
+ if remote_checksum is None:
+ raise common.InvalidResponse(
+ response,
+ _UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(metadata_key),
+ self._get_headers(response),
+ )
+ local_checksum = _helpers.prepare_checksum_digest(
+ self._checksum_object.digest()
+ )
+ if local_checksum != remote_checksum:
+ raise common.DataCorruption(
+ response,
+ _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
+ self._checksum_type.upper(), local_checksum, remote_checksum
+ ),
+ )
+
+ def transmit_next_chunk(self, transport, timeout=None):
+ """Transmit the next chunk of the resource to be uploaded.
+
+ If the current upload was initiated with ``stream_final=False``,
+ this method will dynamically determine if the upload has completed.
+ The upload will be considered complete if the stream produces
+ fewer than :attr:`chunk_size` bytes when a chunk is read from it.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _prepare_recover_request(self):
+ """Prepare the contents of HTTP request to recover from failure.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O. This is based on the `sans-I/O`_ philosophy.
+
+ We assume that the :attr:`resumable_url` is set (i.e. the only way
+ the upload can end up :attr:`invalid` is if it has been initiated.
+
+ Returns:
+ Tuple[str, str, NoneType, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always PUT)
+ * the URL for the request
+ * the body of the request (always :data:`None`)
+ * headers for the request
+
+ The headers **do not** incorporate the ``_headers`` on the
+ current instance.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ headers = {_helpers.CONTENT_RANGE_HEADER: "bytes */*"}
+ return _PUT, self.resumable_url, None, headers
+
+ def _process_recover_response(self, response):
+ """Process the response from an HTTP request to recover from failure.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 308.
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is 308 and the ``range`` header is not of the form
+ ``bytes 0-{end}``.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ _helpers.require_status_code(
+ response, (http.client.PERMANENT_REDIRECT,), self._get_status_code
+ )
+ headers = self._get_headers(response)
+ if _helpers.RANGE_HEADER in headers:
+ bytes_range = headers[_helpers.RANGE_HEADER]
+ match = _BYTES_RANGE_RE.match(bytes_range)
+ if match is None:
+ raise common.InvalidResponse(
+ response,
+ 'Unexpected "range" header',
+ bytes_range,
+ 'Expected to be of the form "bytes=0-{end}"',
+ )
+ self._bytes_uploaded = int(match.group("end_byte")) + 1
+ else:
+ # In this case, the upload has not "begun".
+ self._bytes_uploaded = 0
+
+ self._stream.seek(self._bytes_uploaded)
+ self._invalid = False
+
+ def recover(self, transport):
+ """Recover from a failure.
+
+ This method should be used when a :class:`ResumableUpload` is in an
+ :attr:`~ResumableUpload.invalid` state due to a request failure.
+
+ This will verify the progress with the server and make sure the
+ current upload is in a valid state before :meth:`transmit_next_chunk`
+ can be used again.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class XMLMPUContainer(UploadBase):
+ """Initiate and close an upload using the XML MPU API.
+
+ An XML MPU sends an initial request and then receives an upload ID.
+ Using the upload ID, the upload is then done in numbered parts and the
+ parts can be uploaded concurrently.
+
+ In order to avoid concurrency issues with this container object, the
+ uploading of individual parts is handled separately, by XMLMPUPart objects
+ spawned from this container class. The XMLMPUPart objects are not
+ necessarily in the same process as the container, so they do not update the
+ container automatically.
+
+ MPUs are sometimes referred to as "Multipart Uploads", which is ambiguous
+ given the JSON multipart upload, so the abbreviation "MPU" will be used
+ throughout.
+
+ See: https://cloud.google.com/storage/docs/multipart-uploads
+
+ Args:
+ upload_url (str): The URL of the object (without query parameters). The
+ initiate, PUT, and finalization requests will all use this URL, with
+ varying query parameters.
+ filename (str): The name (path) of the file to upload.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with every request.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ upload_id (Optional(str)): The ID of the upload from the initialization
+ response.
+ """
+
+ def __init__(self, upload_url, filename, headers=None, upload_id=None):
+ super().__init__(upload_url, headers=headers)
+ self._filename = filename
+ self._upload_id = upload_id
+ self._parts = {}
+
+ @property
+ def upload_id(self):
+ return self._upload_id
+
+ def register_part(self, part_number, etag):
+ """Register an uploaded part by part number and corresponding etag.
+
+ XMLMPUPart objects represent individual parts, and their part number
+ and etag can be registered to the container object with this method
+ and therefore incorporated in the finalize() call to finish the upload.
+
+ This method accepts part_number and etag, but not XMLMPUPart objects
+ themselves, to reduce the complexity involved in running XMLMPUPart
+ uploads in separate processes.
+
+ Args:
+ part_number (int): The part number. Parts are assembled into the
+ final uploaded object with finalize() in order of their part
+ numbers.
+ etag (str): The etag included in the server response after upload.
+ """
+ self._parts[part_number] = etag
+
+ def _prepare_initiate_request(self, content_type):
+ """Prepare the contents of HTTP request to initiate upload.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the current upload has already been initiated.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.upload_id is not None:
+ raise ValueError("This upload has already been initiated.")
+
+ initiate_url = self.upload_url + _MPU_INITIATE_QUERY
+
+ headers = {
+ **self._headers,
+ _CONTENT_TYPE_HEADER: content_type,
+ }
+ return _POST, initiate_url, None, headers
+
+ def _process_initiate_response(self, response):
+ """Process the response from an HTTP request that initiated the upload.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ This method takes the URL from the ``Location`` header and stores it
+ for future use. Within that URL, we assume the ``upload_id`` query
+ parameter has been included, but we do not check.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ _helpers.require_status_code(response, (http.client.OK,), self._get_status_code)
+ root = ElementTree.fromstring(response.text)
+ self._upload_id = root.find(_S3_COMPAT_XML_NAMESPACE + _UPLOAD_ID_NODE).text
+
+ def initiate(
+ self,
+ transport,
+ content_type,
+ timeout=None,
+ ):
+ """Initiate an MPU and record the upload ID.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _prepare_finalize_request(self):
+ """Prepare the contents of an HTTP request to finalize the upload.
+
+ All of the parts must be registered before calling this method.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always POST)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the upload has not been initiated.
+ """
+ if self.upload_id is None:
+ raise ValueError("This upload has not yet been initiated.")
+
+ final_query = _MPU_FINAL_QUERY_TEMPLATE.format(upload_id=self._upload_id)
+ finalize_url = self.upload_url + final_query
+ final_xml_root = ElementTree.Element("CompleteMultipartUpload")
+ for part_number, etag in self._parts.items():
+ part = ElementTree.SubElement(final_xml_root, "Part") # put in a loop
+ ElementTree.SubElement(part, "PartNumber").text = str(part_number)
+ ElementTree.SubElement(part, "ETag").text = etag
+ payload = ElementTree.tostring(final_xml_root)
+ return _POST, finalize_url, payload, self._headers
+
+ def _process_finalize_response(self, response):
+ """Process the response from an HTTP request that finalized the upload.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+
+ _helpers.require_status_code(response, (http.client.OK,), self._get_status_code)
+ self._finished = True
+
+ def finalize(
+ self,
+ transport,
+ timeout=None,
+ ):
+ """Finalize an MPU request with all the parts.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _prepare_cancel_request(self):
+ """Prepare the contents of an HTTP request to cancel the upload.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always DELETE)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ Raises:
+ ValueError: If the upload has not been initiated.
+ """
+ if self.upload_id is None:
+ raise ValueError("This upload has not yet been initiated.")
+
+ cancel_query = _MPU_FINAL_QUERY_TEMPLATE.format(upload_id=self._upload_id)
+ cancel_url = self.upload_url + cancel_query
+ return _DELETE, cancel_url, None, self._headers
+
+ def _process_cancel_response(self, response):
+ """Process the response from an HTTP request that canceled the upload.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 204.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+
+ _helpers.require_status_code(
+ response, (http.client.NO_CONTENT,), self._get_status_code
+ )
+
+ def cancel(
+ self,
+ transport,
+ timeout=None,
+ ):
+ """Cancel an MPU request and permanently delete any uploaded parts.
+
+ This cannot be undone.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+
+class XMLMPUPart(UploadBase):
+ """Upload a single part of an existing XML MPU container.
+
+ An XML MPU sends an initial request and then receives an upload ID.
+ Using the upload ID, the upload is then done in numbered parts and the
+ parts can be uploaded concurrently.
+
+ In order to avoid concurrency issues with the container object, the
+ uploading of individual parts is handled separately by multiple objects
+ of this class. Once a part is uploaded, it can be registered with the
+ container with `container.register_part(part.part_number, part.etag)`.
+
+ MPUs are sometimes referred to as "Multipart Uploads", which is ambiguous
+ given the JSON multipart upload, so the abbreviation "MPU" will be used
+ throughout.
+
+ See: https://cloud.google.com/storage/docs/multipart-uploads
+
+ Args:
+ upload_url (str): The URL of the object (without query parameters).
+ upload_id (str): The ID of the upload from the initialization response.
+ filename (str): The name (path) of the file to upload.
+ start (int): The byte index of the beginning of the part.
+ end (int): The byte index of the end of the part.
+ part_number (int): The part number. Part numbers will be assembled in
+ sequential order when the container is finalized.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with every request.
+ checksum (Optional([str])): The type of checksum to compute to verify
+ the integrity of the object. The request headers will be amended
+ to include the computed value. Supported values are "md5", "crc32c"
+ and None. The default is None.
+
+ Attributes:
+ upload_url (str): The URL of the object (without query parameters).
+ upload_id (str): The ID of the upload from the initialization response.
+ filename (str): The name (path) of the file to upload.
+ start (int): The byte index of the beginning of the part.
+ end (int): The byte index of the end of the part.
+ part_number (int): The part number. Part numbers will be assembled in
+ sequential order when the container is finalized.
+ etag (Optional(str)): The etag returned by the service after upload.
+ """
+
+ def __init__(
+ self,
+ upload_url,
+ upload_id,
+ filename,
+ start,
+ end,
+ part_number,
+ headers=None,
+ checksum=None,
+ ):
+ super().__init__(upload_url, headers=headers)
+ self._filename = filename
+ self._start = start
+ self._end = end
+ self._upload_id = upload_id
+ self._part_number = part_number
+ self._etag = None
+ self._checksum_type = checksum
+ self._checksum_object = None
+
+ @property
+ def part_number(self):
+ return self._part_number
+
+ @property
+ def upload_id(self):
+ return self._upload_id
+
+ @property
+ def filename(self):
+ return self._filename
+
+ @property
+ def etag(self):
+ return self._etag
+
+ @property
+ def start(self):
+ return self._start
+
+ @property
+ def end(self):
+ return self._end
+
+ def _prepare_upload_request(self):
+ """Prepare the contents of HTTP request to upload a part.
+
+ This is everything that must be done before a request that doesn't
+ require network I/O. This is based on the `sans-I/O`_ philosophy.
+
+ For the time being, this **does require** some form of I/O to read
+ a part from ``stream`` (via :func:`get_part_payload`). However, this
+ will (almost) certainly not be network I/O.
+
+ Returns:
+ Tuple[str, str, bytes, Mapping[str, str]]: The quadruple
+
+ * HTTP verb for the request (always PUT)
+ * the URL for the request
+ * the body of the request
+ * headers for the request
+
+ The headers incorporate the ``_headers`` on the current instance.
+
+ Raises:
+ ValueError: If the current upload has finished.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ if self.finished:
+ raise ValueError("This part has already been uploaded.")
+
+ with open(self._filename, "br") as f:
+ f.seek(self._start)
+ payload = f.read(self._end - self._start)
+
+ self._checksum_object = _helpers._get_checksum_object(self._checksum_type)
+ if self._checksum_object is not None:
+ self._checksum_object.update(payload)
+
+ part_query = _MPU_PART_QUERY_TEMPLATE.format(
+ part=self._part_number, upload_id=self._upload_id
+ )
+ upload_url = self.upload_url + part_query
+ return _PUT, upload_url, payload, self._headers
+
+ def _process_upload_response(self, response):
+ """Process the response from an HTTP request.
+
+ This is everything that must be done after a request that doesn't
+ require network I/O (or other I/O). This is based on the `sans-I/O`_
+ philosophy.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200 or the response is missing data.
+
+ .. _sans-I/O: https://sans-io.readthedocs.io/
+ """
+ _helpers.require_status_code(
+ response,
+ (http.client.OK,),
+ self._get_status_code,
+ )
+
+ self._validate_checksum(response)
+
+ etag = _helpers.header_required(response, "etag", self._get_headers)
+ self._etag = etag
+ self._finished = True
+
+ def upload(
+ self,
+ transport,
+ timeout=None,
+ ):
+ """Upload the part.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Raises:
+ NotImplementedError: Always, since virtual.
+ """
+ raise NotImplementedError("This implementation is virtual.")
+
+ def _validate_checksum(self, response):
+ """Check the computed checksum, if any, against the response headers.
+
+ Args:
+ response (object): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the checksum
+ computed locally and the checksum reported by the remote host do
+ not match.
+ """
+ if self._checksum_type is None:
+ return
+
+ remote_checksum = _helpers._get_uploaded_checksum_from_headers(
+ response, self._get_headers, self._checksum_type
+ )
+
+ if remote_checksum is None:
+ metadata_key = _helpers._get_metadata_key(self._checksum_type)
+ raise common.InvalidResponse(
+ response,
+ _UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(metadata_key),
+ self._get_headers(response),
+ )
+ local_checksum = _helpers.prepare_checksum_digest(
+ self._checksum_object.digest()
+ )
+ if local_checksum != remote_checksum:
+ raise common.DataCorruption(
+ response,
+ _UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
+ self._checksum_type.upper(), local_checksum, remote_checksum
+ ),
+ )
+
+
+def get_boundary():
+ """Get a random boundary for a multipart request.
+
+ Returns:
+ bytes: The boundary used to separate parts of a multipart request.
+ """
+ random_int = random.randrange(sys.maxsize)
+ boundary = _BOUNDARY_FORMAT.format(random_int)
+ # NOTE: Neither % formatting nor .format() are available for byte strings
+ # in Python 3.4, so we must use unicode strings as templates.
+ return boundary.encode("utf-8")
+
+
+def construct_multipart_request(data, metadata, content_type):
+ """Construct a multipart request body.
+
+ Args:
+ data (bytes): The resource content (UTF-8 encoded as bytes)
+ to be uploaded.
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+
+ Returns:
+ Tuple[bytes, bytes]: The multipart request body and the boundary used
+ between each part.
+ """
+ multipart_boundary = get_boundary()
+ json_bytes = json.dumps(metadata).encode("utf-8")
+ content_type = content_type.encode("utf-8")
+ # Combine the two parts into a multipart payload.
+ # NOTE: We'd prefer a bytes template but are restricted by Python 3.4.
+ boundary_sep = _MULTIPART_SEP + multipart_boundary
+ content = (
+ boundary_sep
+ + _MULTIPART_BEGIN
+ + json_bytes
+ + _CRLF
+ + boundary_sep
+ + _CRLF
+ + b"content-type: "
+ + content_type
+ + _CRLF
+ + _CRLF
+ + data # Empty line between headers and body.
+ + _CRLF
+ + boundary_sep
+ + _MULTIPART_SEP
+ )
+
+ return content, multipart_boundary
+
+
+def get_total_bytes(stream):
+ """Determine the total number of bytes in a stream.
+
+ Args:
+ stream (IO[bytes]): The stream (i.e. file-like object).
+
+ Returns:
+ int: The number of bytes.
+ """
+ current_position = stream.tell()
+ # NOTE: ``.seek()`` **should** return the same value that ``.tell()``
+ # returns, but in Python 2, ``file`` objects do not.
+ stream.seek(0, os.SEEK_END)
+ end_position = stream.tell()
+ # Go back to the initial position.
+ stream.seek(current_position)
+
+ return end_position
+
+
+def get_next_chunk(stream, chunk_size, total_bytes):
+ """Get a chunk from an I/O stream.
+
+ The ``stream`` may have fewer bytes remaining than ``chunk_size``
+ so it may not always be the case that
+ ``end_byte == start_byte + chunk_size - 1``.
+
+ Args:
+ stream (IO[bytes]): The stream (i.e. file-like object).
+ chunk_size (int): The size of the chunk to be read from the ``stream``.
+ total_bytes (Optional[int]): The (expected) total number of bytes
+ in the ``stream``.
+
+ Returns:
+ Tuple[int, bytes, str]: Triple of:
+
+ * the start byte index
+ * the content in between the start and end bytes (inclusive)
+ * content range header for the chunk (slice) that has been read
+
+ Raises:
+ ValueError: If ``total_bytes == 0`` but ``stream.read()`` yields
+ non-empty content.
+ ValueError: If there is no data left to consume. This corresponds
+ exactly to the case ``end_byte < start_byte``, which can only
+ occur if ``end_byte == start_byte - 1``.
+ """
+ start_byte = stream.tell()
+ if total_bytes is not None and start_byte + chunk_size >= total_bytes > 0:
+ payload = stream.read(total_bytes - start_byte)
+ else:
+ payload = stream.read(chunk_size)
+ end_byte = stream.tell() - 1
+
+ num_bytes_read = len(payload)
+ if total_bytes is None:
+ if num_bytes_read < chunk_size:
+ # We now **KNOW** the total number of bytes.
+ total_bytes = end_byte + 1
+ elif total_bytes == 0:
+ # NOTE: We also expect ``start_byte == 0`` here but don't check
+ # because ``_prepare_initiate_request()`` requires the
+ # stream to be at the beginning.
+ if num_bytes_read != 0:
+ raise ValueError(
+ "Stream specified as empty, but produced non-empty content."
+ )
+ else:
+ if num_bytes_read == 0:
+ raise ValueError(
+ "Stream is already exhausted. There is no content remaining."
+ )
+
+ content_range = get_content_range(start_byte, end_byte, total_bytes)
+ return start_byte, payload, content_range
+
+
+def get_content_range(start_byte, end_byte, total_bytes):
+ """Convert start, end and total into content range header.
+
+ If ``total_bytes`` is not known, uses "bytes {start}-{end}/*".
+ If we are dealing with an empty range (i.e. ``end_byte < start_byte``)
+ then "bytes */{total}" is used.
+
+ This function **ASSUMES** that if the size is not known, the caller will
+ not also pass an empty range.
+
+ Args:
+ start_byte (int): The start (inclusive) of the byte range.
+ end_byte (int): The end (inclusive) of the byte range.
+ total_bytes (Optional[int]): The number of bytes in the byte
+ range (if known).
+
+ Returns:
+ str: The content range header.
+ """
+ if total_bytes is None:
+ return _RANGE_UNKNOWN_TEMPLATE.format(start_byte, end_byte)
+ elif end_byte < start_byte:
+ return _EMPTY_RANGE_TEMPLATE.format(total_bytes)
+ else:
+ return _CONTENT_RANGE_TEMPLATE.format(start_byte, end_byte, total_bytes)
diff --git a/packages/google-resumable-media/google/resumable_media/common.py b/packages/google-resumable-media/google/resumable_media/common.py
new file mode 100644
index 000000000000..216ba04097a8
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/common.py
@@ -0,0 +1,179 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Common utilities for Google Media Downloads and Resumable Uploads.
+
+Includes custom exception types, useful constants and shared helpers.
+"""
+
+import http.client
+
+_SLEEP_RETRY_ERROR_MSG = (
+ "At most one of `max_cumulative_retry` and `max_retries` can be specified."
+)
+
+UPLOAD_CHUNK_SIZE = 262144 # 256 * 1024
+"""int: Chunks in a resumable upload must come in multiples of 256 KB."""
+
+PERMANENT_REDIRECT = http.client.PERMANENT_REDIRECT # type: ignore
+"""int: Permanent redirect status code.
+
+.. note::
+ This is a backward-compatibility alias.
+
+It is used by Google services to indicate some (but not all) of
+a resumable upload has been completed.
+
+For more information, see `RFC 7238`_.
+
+.. _RFC 7238: https://tools.ietf.org/html/rfc7238
+"""
+
+TOO_MANY_REQUESTS = http.client.TOO_MANY_REQUESTS
+"""int: Status code indicating rate-limiting.
+
+.. note::
+ This is a backward-compatibility alias.
+
+For more information, see `RFC 6585`_.
+
+.. _RFC 6585: https://tools.ietf.org/html/rfc6585#section-4
+"""
+
+MAX_SLEEP = 64.0
+"""float: Maximum amount of time allowed between requests.
+
+Used during the retry process for sleep after a failed request.
+Chosen since it is the power of two nearest to one minute.
+"""
+
+MAX_CUMULATIVE_RETRY = 600.0
+"""float: Maximum total sleep time allowed during retry process.
+
+This is provided (10 minutes) as a default. When the cumulative sleep
+exceeds this limit, no more retries will occur.
+"""
+
+RETRYABLE = (
+ http.client.TOO_MANY_REQUESTS, # 429
+ http.client.REQUEST_TIMEOUT, # 408
+ http.client.INTERNAL_SERVER_ERROR, # 500
+ http.client.BAD_GATEWAY, # 502
+ http.client.SERVICE_UNAVAILABLE, # 503
+ http.client.GATEWAY_TIMEOUT, # 504
+)
+"""iterable: HTTP status codes that indicate a retryable error.
+
+Connection errors are also retried, but are not listed as they are
+exceptions, not status codes.
+"""
+
+
+class InvalidResponse(Exception):
+ """Error class for responses which are not in the correct state.
+
+ Args:
+ response (object): The HTTP response which caused the failure.
+ args (tuple): The positional arguments typically passed to an
+ exception class.
+ """
+
+ def __init__(self, response, *args):
+ super(InvalidResponse, self).__init__(*args)
+ self.response = response
+ """object: The HTTP response object that caused the failure."""
+
+
+class DataCorruption(Exception):
+ """Error class for corrupt media transfers.
+
+ Args:
+ response (object): The HTTP response which caused the failure.
+ args (tuple): The positional arguments typically passed to an
+ exception class.
+ """
+
+ def __init__(self, response, *args):
+ super(DataCorruption, self).__init__(*args)
+ self.response = response
+ """object: The HTTP response object that caused the failure."""
+
+
+class RetryStrategy(object):
+ """Configuration class for retrying failed requests.
+
+ At most one of ``max_cumulative_retry`` and ``max_retries`` can be
+ specified (they are both caps on the total number of retries). If
+ neither are specified, then ``max_cumulative_retry`` is set as
+ :data:`MAX_CUMULATIVE_RETRY`.
+
+ Args:
+ max_sleep (Optional[float]): The maximum amount of time to sleep after
+ a failed request. Default is :attr:`MAX_SLEEP`.
+ max_cumulative_retry (Optional[float]): The maximum **total** amount of
+ time to sleep during retry process.
+ max_retries (Optional[int]): The number of retries to attempt.
+ initial_delay (Optional[float]): The initial delay. Default 1.0 second.
+ muiltiplier (Optional[float]): Exponent of the backoff. Default is 2.0.
+
+ Attributes:
+ max_sleep (float): Maximum amount of time allowed between requests.
+ max_cumulative_retry (Optional[float]): Maximum total sleep time
+ allowed during retry process.
+ max_retries (Optional[int]): The number retries to attempt.
+ initial_delay (Optional[float]): The initial delay. Default 1.0 second.
+ muiltiplier (Optional[float]): Exponent of the backoff. Default is 2.0.
+
+ Raises:
+ ValueError: If both of ``max_cumulative_retry`` and ``max_retries``
+ are passed.
+ """
+
+ def __init__(
+ self,
+ max_sleep=MAX_SLEEP,
+ max_cumulative_retry=None,
+ max_retries=None,
+ initial_delay=1.0,
+ multiplier=2.0,
+ ):
+ if max_cumulative_retry is not None and max_retries is not None:
+ raise ValueError(_SLEEP_RETRY_ERROR_MSG)
+ if max_cumulative_retry is None and max_retries is None:
+ max_cumulative_retry = MAX_CUMULATIVE_RETRY
+
+ self.max_sleep = max_sleep
+ self.max_cumulative_retry = max_cumulative_retry
+ self.max_retries = max_retries
+ self.initial_delay = initial_delay
+ self.multiplier = multiplier
+
+ def retry_allowed(self, total_sleep, num_retries):
+ """Check if another retry is allowed.
+
+ Args:
+ total_sleep (float): With another retry, the amount of sleep that
+ will be accumulated by the caller.
+ num_retries (int): With another retry, the number of retries that
+ will be attempted by the caller.
+
+ Returns:
+ bool: Indicating if another retry is allowed (depending on either
+ the cumulative sleep allowed or the maximum number of retries
+ allowed.
+ """
+ if self.max_cumulative_retry is None:
+ return num_retries <= self.max_retries
+ else:
+ return total_sleep <= self.max_cumulative_retry
diff --git a/packages/google-resumable-media/google/resumable_media/py.typed b/packages/google-resumable-media/google/resumable_media/py.typed
new file mode 100644
index 000000000000..7705b065b00d
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/py.typed
@@ -0,0 +1,2 @@
+# Marker file for PEP 561.
+# The google-resumable_media package uses inline types.
diff --git a/packages/google-resumable-media/google/resumable_media/requests/__init__.py b/packages/google-resumable-media/google/resumable_media/requests/__init__.py
new file mode 100644
index 000000000000..c46a02c4543c
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/requests/__init__.py
@@ -0,0 +1,686 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""``requests`` utilities for Google Media Downloads and Resumable Uploads.
+
+This sub-package assumes callers will use the `requests`_ library
+as transport and `google-auth`_ for sending authenticated HTTP traffic
+with ``requests``.
+
+.. _requests: http://docs.python-requests.org/
+.. _google-auth: https://google-auth.readthedocs.io/
+
+====================
+Authorized Transport
+====================
+
+To use ``google-auth`` and ``requests`` to create an authorized transport
+that has read-only access to Google Cloud Storage (GCS):
+
+.. testsetup:: get-credentials
+
+ import google.auth
+ import google.auth.credentials as creds_mod
+ import mock
+
+ def mock_default(scopes=None):
+ credentials = mock.Mock(spec=creds_mod.Credentials)
+ return credentials, 'mock-project'
+
+ # Patch the ``default`` function on the module.
+ original_default = google.auth.default
+ google.auth.default = mock_default
+
+.. doctest:: get-credentials
+
+ >>> import google.auth
+ >>> import google.auth.transport.requests as tr_requests
+ >>>
+ >>> ro_scope = 'https://www.googleapis.com/auth/devstorage.read_only'
+ >>> credentials, _ = google.auth.default(scopes=(ro_scope,))
+ >>> transport = tr_requests.AuthorizedSession(credentials)
+ >>> transport
+
+
+.. testcleanup:: get-credentials
+
+ # Put back the correct ``default`` function on the module.
+ google.auth.default = original_default
+
+================
+Simple Downloads
+================
+
+To download an object from Google Cloud Storage, construct the media URL
+for the GCS object and download it with an authorized transport that has
+access to the resource:
+
+.. testsetup:: basic-download
+
+ import mock
+ import requests
+ import http.client
+
+ bucket = 'bucket-foo'
+ blob_name = 'file.txt'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response.headers['Content-Length'] = '1364156'
+ fake_content = mock.MagicMock(spec=['__len__'])
+ fake_content.__len__.return_value = 1364156
+ fake_response._content = fake_content
+
+ get_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=get_method, spec=['request'])
+
+.. doctest:: basic-download
+
+ >>> from google.resumable_media.requests import Download
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/download/storage/v1/b/'
+ ... '{bucket}/o/{blob_name}?alt=media')
+ >>> media_url = url_template.format(
+ ... bucket=bucket, blob_name=blob_name)
+ >>>
+ >>> download = Download(media_url)
+ >>> response = download.consume(transport)
+ >>> download.finished
+ True
+ >>> response
+
+ >>> response.headers['Content-Length']
+ '1364156'
+ >>> len(response.content)
+ 1364156
+
+To download only a portion of the bytes in the object,
+specify ``start`` and ``end`` byte positions (both optional):
+
+.. testsetup:: basic-download-with-slice
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import Download
+
+ media_url = 'http://test.invalid'
+ start = 4096
+ end = 8191
+ slice_size = end - start + 1
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.PARTIAL_CONTENT)
+ fake_response.headers['Content-Length'] = '{:d}'.format(slice_size)
+ content_range = 'bytes {:d}-{:d}/1364156'.format(start, end)
+ fake_response.headers['Content-Range'] = content_range
+ fake_content = mock.MagicMock(spec=['__len__'])
+ fake_content.__len__.return_value = slice_size
+ fake_response._content = fake_content
+
+ get_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=get_method, spec=['request'])
+
+.. doctest:: basic-download-with-slice
+
+ >>> download = Download(media_url, start=4096, end=8191)
+ >>> response = download.consume(transport)
+ >>> download.finished
+ True
+ >>> response
+
+ >>> response.headers['Content-Length']
+ '4096'
+ >>> response.headers['Content-Range']
+ 'bytes 4096-8191/1364156'
+ >>> len(response.content)
+ 4096
+
+=================
+Chunked Downloads
+=================
+
+For very large objects or objects of unknown size, it may make more sense
+to download the object in chunks rather than all at once. This can be done
+to avoid dropped connections with a poor internet connection or can allow
+multiple chunks to be downloaded in parallel to speed up the total
+download.
+
+A :class:`.ChunkedDownload` uses the same media URL and authorized
+transport that a basic :class:`.Download` would use, but also
+requires a chunk size and a write-able byte ``stream``. The chunk size is used
+to determine how much of the resouce to consume with each request and the
+stream is to allow the resource to be written out (e.g. to disk) without
+having to fit in memory all at once.
+
+.. testsetup:: chunked-download
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ media_url = 'http://test.invalid'
+
+ fifty_mb = 50 * 1024 * 1024
+ one_gb = 1024 * 1024 * 1024
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.PARTIAL_CONTENT)
+ fake_response.headers['Content-Length'] = '{:d}'.format(fifty_mb)
+ content_range = 'bytes 0-{:d}/{:d}'.format(fifty_mb - 1, one_gb)
+ fake_response.headers['Content-Range'] = content_range
+ fake_content_begin = b'The beginning of the chunk...'
+ fake_content = fake_content_begin + b'1' * (fifty_mb - 29)
+ fake_response._content = fake_content
+
+ get_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=get_method, spec=['request'])
+
+.. doctest:: chunked-download
+
+ >>> from google.resumable_media.requests import ChunkedDownload
+ >>>
+ >>> chunk_size = 50 * 1024 * 1024 # 50MB
+ >>> stream = io.BytesIO()
+ >>> download = ChunkedDownload(
+ ... media_url, chunk_size, stream)
+ >>> # Check the state of the download before starting.
+ >>> download.bytes_downloaded
+ 0
+ >>> download.total_bytes is None
+ True
+ >>> response = download.consume_next_chunk(transport)
+ >>> # Check the state of the download after consuming one chunk.
+ >>> download.finished
+ False
+ >>> download.bytes_downloaded # chunk_size
+ 52428800
+ >>> download.total_bytes # 1GB
+ 1073741824
+ >>> response
+
+ >>> response.headers['Content-Length']
+ '52428800'
+ >>> response.headers['Content-Range']
+ 'bytes 0-52428799/1073741824'
+ >>> len(response.content) == chunk_size
+ True
+ >>> stream.seek(0)
+ 0
+ >>> stream.read(29)
+ b'The beginning of the chunk...'
+
+The download will change it's ``finished`` status to :data:`True`
+once the final chunk is consumed. In some cases, the final chunk may
+not be the same size as the other chunks:
+
+.. testsetup:: chunked-download-end
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import ChunkedDownload
+
+ media_url = 'http://test.invalid'
+
+ fifty_mb = 50 * 1024 * 1024
+ one_gb = 1024 * 1024 * 1024
+ stream = mock.Mock(spec=['write'])
+ download = ChunkedDownload(media_url, fifty_mb, stream)
+ download._bytes_downloaded = 20 * fifty_mb
+ download._total_bytes = one_gb
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.PARTIAL_CONTENT)
+ slice_size = one_gb - 20 * fifty_mb
+ fake_response.headers['Content-Length'] = '{:d}'.format(slice_size)
+ content_range = 'bytes {:d}-{:d}/{:d}'.format(
+ 20 * fifty_mb, one_gb - 1, one_gb)
+ fake_response.headers['Content-Range'] = content_range
+ fake_content = mock.MagicMock(spec=['__len__'])
+ fake_content.__len__.return_value = slice_size
+ fake_response._content = fake_content
+
+ get_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=get_method, spec=['request'])
+
+.. doctest:: chunked-download-end
+
+ >>> # The state of the download in progress.
+ >>> download.finished
+ False
+ >>> download.bytes_downloaded # 20 chunks at 50MB
+ 1048576000
+ >>> download.total_bytes # 1GB
+ 1073741824
+ >>> response = download.consume_next_chunk(transport)
+ >>> # The state of the download after consuming the final chunk.
+ >>> download.finished
+ True
+ >>> download.bytes_downloaded == download.total_bytes
+ True
+ >>> response
+
+ >>> response.headers['Content-Length']
+ '25165824'
+ >>> response.headers['Content-Range']
+ 'bytes 1048576000-1073741823/1073741824'
+ >>> len(response.content) < download.chunk_size
+ True
+
+In addition, a :class:`.ChunkedDownload` can also take optional
+``start`` and ``end`` byte positions.
+
+Usually, no checksum is returned with a chunked download. Even if one is returned,
+it is not validated. If you need to validate the checksum, you can do so
+by buffering the chunks and validating the checksum against the completed download.
+
+==============
+Simple Uploads
+==============
+
+Among the three supported upload classes, the simplest is
+:class:`.SimpleUpload`. A simple upload should be used when the resource
+being uploaded is small and when there is no metadata (other than the name)
+associated with the resource.
+
+.. testsetup:: simple-upload
+
+ import json
+
+ import mock
+ import requests
+ import http.client
+
+ bucket = 'some-bucket'
+ blob_name = 'file.txt'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ payload = {
+ 'bucket': bucket,
+ 'contentType': 'text/plain',
+ 'md5Hash': 'M0XLEsX9/sMdiI+4pB4CAQ==',
+ 'name': blob_name,
+ 'size': '27',
+ }
+ fake_response._content = json.dumps(payload).encode('utf-8')
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+.. doctest:: simple-upload
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> from google.resumable_media.requests import SimpleUpload
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?'
+ ... 'uploadType=media&'
+ ... 'name={blob_name}')
+ >>> upload_url = url_template.format(
+ ... bucket=bucket, blob_name=blob_name)
+ >>>
+ >>> upload = SimpleUpload(upload_url)
+ >>> data = b'Some not too large content.'
+ >>> content_type = 'text/plain'
+ >>> response = upload.transmit(transport, data, content_type)
+ >>> upload.finished
+ True
+ >>> response
+
+ >>> json_response = response.json()
+ >>> json_response['bucket'] == bucket
+ True
+ >>> json_response['name'] == blob_name
+ True
+ >>> json_response['contentType'] == content_type
+ True
+ >>> json_response['md5Hash']
+ 'M0XLEsX9/sMdiI+4pB4CAQ=='
+ >>> int(json_response['size']) == len(data)
+ True
+
+In the rare case that an upload fails, an :exc:`.InvalidResponse`
+will be raised:
+
+.. testsetup:: simple-upload-fail
+
+ import time
+
+ import mock
+ import requests
+ import http.client
+
+ from google import resumable_media
+ from google.resumable_media import _helpers
+ from google.resumable_media.requests import SimpleUpload as constructor
+
+ upload_url = 'http://test.invalid'
+ data = b'Some not too large content.'
+ content_type = 'text/plain'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.SERVICE_UNAVAILABLE)
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+ time_sleep = time.sleep
+ def dont_sleep(seconds):
+ raise RuntimeError('No sleep', seconds)
+
+ def SimpleUpload(*args, **kwargs):
+ upload = constructor(*args, **kwargs)
+ # Mock the cumulative sleep to avoid retries (and `time.sleep()`).
+ upload._retry_strategy = resumable_media.RetryStrategy(
+ max_cumulative_retry=-1.0)
+ return upload
+
+ time.sleep = dont_sleep
+
+.. doctest:: simple-upload-fail
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> upload = SimpleUpload(upload_url)
+ >>> error = None
+ >>> try:
+ ... upload.transmit(transport, data, content_type)
+ ... except resumable_media.InvalidResponse as caught_exc:
+ ... error = caught_exc
+ ...
+ >>> error
+ InvalidResponse('Request failed with status code', 503,
+ 'Expected one of', )
+ >>> error.response
+
+ >>>
+ >>> upload.finished
+ True
+
+.. testcleanup:: simple-upload-fail
+
+ # Put back the correct ``sleep`` function on the ``time`` module.
+ time.sleep = time_sleep
+
+Even in the case of failure, we see that the upload is
+:attr:`~.SimpleUpload.finished`, i.e. it cannot be re-used.
+
+=================
+Multipart Uploads
+=================
+
+After the simple upload, the :class:`.MultipartUpload` can be used to
+achieve essentially the same task. However, a multipart upload allows some
+metadata about the resource to be sent along as well. (This is the "multi":
+we send a first part with the metadata and a second part with the actual
+bytes in the resource.)
+
+Usage is similar to the simple upload, but :meth:`~.MultipartUpload.transmit`
+accepts an extra required argument: ``metadata``.
+
+.. testsetup:: multipart-upload
+
+ import json
+
+ import mock
+ import requests
+ import http.client
+
+ bucket = 'some-bucket'
+ blob_name = 'file.txt'
+ data = b'Some not too large content.'
+ content_type = 'text/plain'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ payload = {
+ 'bucket': bucket,
+ 'name': blob_name,
+ 'metadata': {'color': 'grurple'},
+ }
+ fake_response._content = json.dumps(payload).encode('utf-8')
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+.. doctest:: multipart-upload
+
+ >>> from google.resumable_media.requests import MultipartUpload
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?'
+ ... 'uploadType=multipart')
+ >>> upload_url = url_template.format(bucket=bucket)
+ >>>
+ >>> upload = MultipartUpload(upload_url)
+ >>> metadata = {
+ ... 'name': blob_name,
+ ... 'metadata': {
+ ... 'color': 'grurple',
+ ... },
+ ... }
+ >>> response = upload.transmit(transport, data, metadata, content_type)
+ >>> upload.finished
+ True
+ >>> response
+
+ >>> json_response = response.json()
+ >>> json_response['bucket'] == bucket
+ True
+ >>> json_response['name'] == blob_name
+ True
+ >>> json_response['metadata'] == metadata['metadata']
+ True
+
+As with the simple upload, in the case of failure an :exc:`.InvalidResponse`
+is raised, enclosing the :attr:`~.InvalidResponse.response` that caused
+the failure and the ``upload`` object cannot be re-used after a failure.
+
+=================
+Resumable Uploads
+=================
+
+A :class:`.ResumableUpload` deviates from the other two upload classes:
+it transmits a resource over the course of multiple requests. This
+is intended to be used in cases where:
+
+* the size of the resource is not known (i.e. it is generated on the fly)
+* requests must be short-lived
+* the client has request **size** limitations
+* the resource is too large to fit into memory
+
+In general, a resource should be sent in a **single** request to avoid
+latency and reduce QPS. See `GCS best practices`_ for more things to
+consider when using a resumable upload.
+
+.. _GCS best practices: https://cloud.google.com/storage/docs/\
+ best-practices#uploading
+
+After creating a :class:`.ResumableUpload` instance, a
+**resumable upload session** must be initiated to let the server know that
+a series of chunked upload requests will be coming and to obtain an
+``upload_id`` for the session. In contrast to the other two upload classes,
+:meth:`~.ResumableUpload.initiate` takes a byte ``stream`` as input rather
+than raw bytes as ``data``. This can be a file object, a :class:`~io.BytesIO`
+object or any other stream implementing the same interface.
+
+.. testsetup:: resumable-initiate
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ bucket = 'some-bucket'
+ blob_name = 'file.txt'
+ data = b'Some resumable bytes.'
+ content_type = 'text/plain'
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response._content = b''
+ upload_id = 'ABCdef189XY_super_serious'
+ resumable_url_template = (
+ 'https://www.googleapis.com/upload/storage/v1/b/{bucket}'
+ '/o?uploadType=resumable&upload_id={upload_id}')
+ resumable_url = resumable_url_template.format(
+ bucket=bucket, upload_id=upload_id)
+ fake_response.headers['location'] = resumable_url
+ fake_response.headers['x-guploader-uploadid'] = upload_id
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+.. doctest:: resumable-initiate
+
+ >>> from google.resumable_media.requests import ResumableUpload
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?'
+ ... 'uploadType=resumable')
+ >>> upload_url = url_template.format(bucket=bucket)
+ >>>
+ >>> chunk_size = 1024 * 1024 # 1MB
+ >>> upload = ResumableUpload(upload_url, chunk_size)
+ >>> stream = io.BytesIO(data)
+ >>> # The upload doesn't know how "big" it is until seeing a stream.
+ >>> upload.total_bytes is None
+ True
+ >>> metadata = {'name': blob_name}
+ >>> response = upload.initiate(transport, stream, metadata, content_type)
+ >>> response
+
+ >>> upload.resumable_url == response.headers['Location']
+ True
+ >>> upload.total_bytes == len(data)
+ True
+ >>> upload_id = response.headers['X-GUploader-UploadID']
+ >>> upload_id
+ 'ABCdef189XY_super_serious'
+ >>> upload.resumable_url == upload_url + '&upload_id=' + upload_id
+ True
+
+Once a :class:`.ResumableUpload` has been initiated, the resource is
+transmitted in chunks until completion:
+
+.. testsetup:: resumable-transmit
+
+ import io
+ import json
+
+ import mock
+ import requests
+ import http.client
+
+ from google import resumable_media
+ import google.resumable_media.requests.upload as upload_mod
+
+ data = b'01234567891'
+ stream = io.BytesIO(data)
+ # Create an "already initiated" upload.
+ upload_url = 'http://test.invalid'
+ chunk_size = 256 * 1024 # 256KB
+ upload = upload_mod.ResumableUpload(upload_url, chunk_size)
+ upload._resumable_url = 'http://test.invalid?upload_id=mocked'
+ upload._stream = stream
+ upload._content_type = 'text/plain'
+ upload._total_bytes = len(data)
+
+ # After-the-fact update the chunk size so that len(data)
+ # is split into three.
+ upload._chunk_size = 4
+ # Make three fake responses.
+ fake_response0 = requests.Response()
+ fake_response0.status_code = http.client.PERMANENT_REDIRECT
+ fake_response0.headers['range'] = 'bytes=0-3'
+
+ fake_response1 = requests.Response()
+ fake_response1.status_code = http.client.PERMANENT_REDIRECT
+ fake_response1.headers['range'] = 'bytes=0-7'
+
+ fake_response2 = requests.Response()
+ fake_response2.status_code = int(http.client.OK)
+ bucket = 'some-bucket'
+ blob_name = 'file.txt'
+ payload = {
+ 'bucket': bucket,
+ 'name': blob_name,
+ 'size': '{:d}'.format(len(data)),
+ }
+ fake_response2._content = json.dumps(payload).encode('utf-8')
+
+ # Use the fake responses to mock a transport.
+ responses = [fake_response0, fake_response1, fake_response2]
+ put_method = mock.Mock(side_effect=responses, spec=[])
+ transport = mock.Mock(request=put_method, spec=['request'])
+
+.. doctest:: resumable-transmit
+
+ >>> response0 = upload.transmit_next_chunk(transport)
+ >>> response0
+
+ >>> upload.finished
+ False
+ >>> upload.bytes_uploaded == upload.chunk_size
+ True
+ >>>
+ >>> response1 = upload.transmit_next_chunk(transport)
+ >>> response1
+
+ >>> upload.finished
+ False
+ >>> upload.bytes_uploaded == 2 * upload.chunk_size
+ True
+ >>>
+ >>> response2 = upload.transmit_next_chunk(transport)
+ >>> response2
+
+ >>> upload.finished
+ True
+ >>> upload.bytes_uploaded == upload.total_bytes
+ True
+ >>> json_response = response2.json()
+ >>> json_response['bucket'] == bucket
+ True
+ >>> json_response['name'] == blob_name
+ True
+"""
+
+from google.resumable_media.requests.download import ChunkedDownload
+from google.resumable_media.requests.download import Download
+from google.resumable_media.requests.upload import MultipartUpload
+from google.resumable_media.requests.download import RawChunkedDownload
+from google.resumable_media.requests.download import RawDownload
+from google.resumable_media.requests.upload import ResumableUpload
+from google.resumable_media.requests.upload import SimpleUpload
+from google.resumable_media.requests.upload import XMLMPUContainer
+from google.resumable_media.requests.upload import XMLMPUPart
+
+__all__ = [
+ "ChunkedDownload",
+ "Download",
+ "MultipartUpload",
+ "RawChunkedDownload",
+ "RawDownload",
+ "ResumableUpload",
+ "SimpleUpload",
+ "XMLMPUContainer",
+ "XMLMPUPart",
+]
diff --git a/packages/google-resumable-media/google/resumable_media/requests/_request_helpers.py b/packages/google-resumable-media/google/resumable_media/requests/_request_helpers.py
new file mode 100644
index 000000000000..051f0bae075d
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/requests/_request_helpers.py
@@ -0,0 +1,180 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Shared utilities used by both downloads and uploads.
+
+This utilities are explicitly catered to ``requests``-like transports.
+"""
+
+import http.client
+import requests.exceptions
+import urllib3.exceptions # type: ignore
+
+import time
+
+from google.resumable_media import common
+from google.resumable_media import _helpers
+
+_DEFAULT_RETRY_STRATEGY = common.RetryStrategy()
+_SINGLE_GET_CHUNK_SIZE = 8192
+# The number of seconds to wait to establish a connection
+# (connect() call on socket). Avoid setting this to a multiple of 3 to not
+# Align with TCP Retransmission timing. (typically 2.5-3s)
+_DEFAULT_CONNECT_TIMEOUT = 61
+# The number of seconds to wait between bytes sent from the server.
+_DEFAULT_READ_TIMEOUT = 60
+
+_CONNECTION_ERROR_CLASSES = (
+ http.client.BadStatusLine,
+ http.client.IncompleteRead,
+ http.client.ResponseNotReady,
+ requests.exceptions.ConnectionError,
+ requests.exceptions.ChunkedEncodingError,
+ requests.exceptions.Timeout,
+ urllib3.exceptions.PoolError,
+ urllib3.exceptions.ProtocolError,
+ urllib3.exceptions.SSLError,
+ urllib3.exceptions.TimeoutError,
+ ConnectionError, # Python 3.x only, superclass of ConnectionResetError.
+)
+
+
+class RequestsMixin(object):
+ """Mix-in class implementing ``requests``-specific behavior.
+
+ These are methods that are more general purpose, with implementations
+ specific to the types defined in ``requests``.
+ """
+
+ @staticmethod
+ def _get_status_code(response):
+ """Access the status code from an HTTP response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Returns:
+ int: The status code.
+ """
+ return response.status_code
+
+ @staticmethod
+ def _get_headers(response):
+ """Access the headers from an HTTP response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Returns:
+ ~requests.structures.CaseInsensitiveDict: The header mapping (keys
+ are case-insensitive).
+ """
+ return response.headers
+
+ @staticmethod
+ def _get_body(response):
+ """Access the response body from an HTTP response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Returns:
+ bytes: The body of the ``response``.
+ """
+ return response.content
+
+
+class RawRequestsMixin(RequestsMixin):
+ @staticmethod
+ def _get_body(response):
+ """Access the response body from an HTTP response.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Returns:
+ bytes: The body of the ``response``.
+ """
+ if response._content is False:
+ response._content = b"".join(
+ response.raw.stream(_SINGLE_GET_CHUNK_SIZE, decode_content=False)
+ )
+ response._content_consumed = True
+ return response._content
+
+
+def wait_and_retry(func, get_status_code, retry_strategy):
+ """Attempts to retry a call to ``func`` until success.
+
+ Expects ``func`` to return an HTTP response and uses ``get_status_code``
+ to check if the response is retry-able.
+
+ ``func`` is expected to raise a failure status code as a
+ common.InvalidResponse, at which point this method will check the code
+ against the common.RETRIABLE list of retriable status codes.
+
+ Will retry until :meth:`~.RetryStrategy.retry_allowed` (on the current
+ ``retry_strategy``) returns :data:`False`. Uses
+ :func:`_helpers.calculate_retry_wait` to double the wait time (with jitter)
+ after each attempt.
+
+ Args:
+ func (Callable): A callable that takes no arguments and produces
+ an HTTP response which will be checked as retry-able.
+ get_status_code (Callable[Any, int]): Helper to get a status code
+ from a response.
+ retry_strategy (~google.resumable_media.common.RetryStrategy): The
+ strategy to use if the request fails and must be retried.
+
+ Returns:
+ object: The return value of ``func``.
+ """
+ total_sleep = 0.0
+ num_retries = 0
+ # base_wait will be multiplied by the multiplier on the first retry.
+ base_wait = float(retry_strategy.initial_delay) / retry_strategy.multiplier
+
+ # Set the retriable_exception_type if possible. We expect requests to be
+ # present here and the transport to be using requests.exceptions errors,
+ # but due to loose coupling with the transport layer we can't guarantee it.
+
+ while True: # return on success or when retries exhausted.
+ error = None
+ try:
+ response = func()
+ except _CONNECTION_ERROR_CLASSES as e:
+ error = e # Fall through to retry, if there are retries left.
+ except common.InvalidResponse as e:
+ # An InvalidResponse is only retriable if its status code matches.
+ # The `process_response()` method on a Download or Upload method
+ # will convert the status code into an exception.
+ if get_status_code(e.response) in common.RETRYABLE:
+ error = e # Fall through to retry, if there are retries left.
+ else:
+ raise # If the status code is not retriable, raise w/o retry.
+ else:
+ return response
+
+ base_wait, wait_time = _helpers.calculate_retry_wait(
+ base_wait, retry_strategy.max_sleep, retry_strategy.multiplier
+ )
+ num_retries += 1
+ total_sleep += wait_time
+
+ # Check if (another) retry is allowed. If retries are exhausted and
+ # no acceptable response was received, raise the retriable error.
+ if not retry_strategy.retry_allowed(total_sleep, num_retries):
+ raise error
+
+ time.sleep(wait_time)
diff --git a/packages/google-resumable-media/google/resumable_media/requests/download.py b/packages/google-resumable-media/google/resumable_media/requests/download.py
new file mode 100644
index 000000000000..be017f549c27
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/requests/download.py
@@ -0,0 +1,739 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Support for downloading media from Google APIs."""
+
+import urllib3.response # type: ignore
+import http
+
+from google.resumable_media import _download
+from google.resumable_media import common
+from google.resumable_media import _helpers
+from google.resumable_media.requests import _request_helpers
+
+
+_CHECKSUM_MISMATCH = """\
+Checksum mismatch while downloading:
+
+ {}
+
+The X-Goog-Hash header indicated an {checksum_type} checksum of:
+
+ {}
+
+but the actual {checksum_type} checksum of the downloaded contents was:
+
+ {}
+"""
+
+_STREAM_SEEK_ERROR = """\
+Incomplete download for:
+{}
+Error writing to stream while handling a gzip-compressed file download.
+Please restart the download.
+"""
+
+_RESPONSE_HEADERS_INFO = """\
+
+The X-Goog-Stored-Content-Length is {}. The X-Goog-Stored-Content-Encoding is {}.
+
+The download request read {} bytes of data.
+If the download was incomplete, please check the network connection and restart the download.
+"""
+
+
+class Download(_request_helpers.RequestsMixin, _download.Download):
+ """Helper to manage downloading a resource from a Google API.
+
+ "Slices" of the resource can be retrieved by specifying a range
+ with ``start`` and / or ``end``. However, in typical usage, neither
+ ``start`` nor ``end`` is expected to be provided.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ the downloaded resource can be written to.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, but ``end`` is provided, will download from the
+ beginning to ``end`` of the media.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, but ``start`` is provided, will download from the
+ ``start`` to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The response headers must contain
+ a checksum of the requested type. If the headers lack an
+ appropriate checksum (for instance in the case of transcoded or
+ ranged downloads where the remote service does not know the
+ correct checksum) an INFO-level log will be emitted. Supported
+ values are "md5", "crc32c" and None. The default is "md5".
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ """
+
+ def _write_to_stream(self, response):
+ """Write response body to a write-able stream.
+
+ .. note:
+
+ This method assumes that the ``_stream`` attribute is set on the
+ current download.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the download's
+ checksum doesn't agree with server-computed checksum.
+ """
+
+ # Retrieve the expected checksum only once for the download request,
+ # then compute and validate the checksum when the full download completes.
+ # Retried requests are range requests, and there's no way to detect
+ # data corruption for that byte range alone.
+ if self._expected_checksum is None and self._checksum_object is None:
+ # `_get_expected_checksum()` may return None even if a checksum was
+ # requested, in which case it will emit an info log _MISSING_CHECKSUM.
+ # If an invalid checksum type is specified, this will raise ValueError.
+ expected_checksum, checksum_object = _helpers._get_expected_checksum(
+ response, self._get_headers, self.media_url, checksum_type=self.checksum
+ )
+ self._expected_checksum = expected_checksum
+ self._checksum_object = checksum_object
+ else:
+ expected_checksum = self._expected_checksum
+ checksum_object = self._checksum_object
+
+ with response:
+ # NOTE: In order to handle compressed streams gracefully, we try
+ # to insert our checksum object into the decompression stream. If
+ # the stream is indeed compressed, this will delegate the checksum
+ # object to the decoder and return a _DoNothingHash here.
+ local_checksum_object = _add_decoder(response.raw, checksum_object)
+ body_iter = response.iter_content(
+ chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
+ )
+ for chunk in body_iter:
+ self._stream.write(chunk)
+ self._bytes_downloaded += len(chunk)
+ local_checksum_object.update(chunk)
+
+ # Don't validate the checksum for partial responses.
+ if (
+ expected_checksum is not None
+ and response.status_code != http.client.PARTIAL_CONTENT
+ ):
+ actual_checksum = _helpers.prepare_checksum_digest(checksum_object.digest())
+
+ if actual_checksum != expected_checksum:
+ headers = self._get_headers(response)
+ x_goog_encoding = headers.get("x-goog-stored-content-encoding")
+ x_goog_length = headers.get("x-goog-stored-content-length")
+ content_length_msg = _RESPONSE_HEADERS_INFO.format(
+ x_goog_length, x_goog_encoding, self._bytes_downloaded
+ )
+ if (
+ x_goog_length
+ and self._bytes_downloaded < int(x_goog_length)
+ and x_goog_encoding != "gzip"
+ ):
+ # The library will attempt to trigger a retry by raising a ConnectionError, if
+ # (a) bytes_downloaded is less than response header x-goog-stored-content-length, and
+ # (b) the object is not gzip-compressed when stored in Cloud Storage.
+ raise ConnectionError(content_length_msg)
+ else:
+ msg = _CHECKSUM_MISMATCH.format(
+ self.media_url,
+ expected_checksum,
+ actual_checksum,
+ checksum_type=self.checksum.upper(),
+ )
+ msg += content_length_msg
+ raise common.DataCorruption(response, msg)
+
+ def consume(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Consume the resource to be downloaded.
+
+ If a ``stream`` is attached to this download, then the downloaded
+ resource will be written to the stream.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the download's
+ checksum doesn't agree with server-computed checksum.
+ ValueError: If the current :class:`Download` has already
+ finished.
+ """
+ method, _, payload, headers = self._prepare_request()
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ request_kwargs = {
+ "data": payload,
+ "headers": headers,
+ "timeout": timeout,
+ }
+ if self._stream is not None:
+ request_kwargs["stream"] = True
+
+ # Assign object generation if generation is specified in the media url.
+ if self._object_generation is None:
+ self._object_generation = _helpers._get_generation_from_url(self.media_url)
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ url = self.media_url
+
+ # To restart an interrupted download, read from the offset of last byte
+ # received using a range request, and set object generation query param.
+ if self._bytes_downloaded > 0:
+ _download.add_bytes_range(
+ (self.start or 0) + self._bytes_downloaded, self.end, self._headers
+ )
+ request_kwargs["headers"] = self._headers
+
+ # Set object generation query param to ensure the same object content is requested.
+ if (
+ self._object_generation is not None
+ and _helpers._get_generation_from_url(self.media_url) is None
+ ):
+ query_param = {"generation": self._object_generation}
+ url = _helpers.add_query_parameters(self.media_url, query_param)
+
+ result = transport.request(method, url, **request_kwargs)
+
+ # If a generation hasn't been specified, and this is the first response we get, let's record the
+ # generation. In future requests we'll specify the generation query param to avoid data races.
+ if self._object_generation is None:
+ self._object_generation = _helpers._parse_generation_header(
+ result, self._get_headers
+ )
+
+ self._process_response(result)
+
+ # With decompressive transcoding, GCS serves back the whole file regardless of the range request,
+ # thus we reset the stream position to the start of the stream.
+ # See: https://cloud.google.com/storage/docs/transcoding#range
+ if self._stream is not None:
+ if _helpers._is_decompressive_transcoding(result, self._get_headers):
+ try:
+ self._stream.seek(0)
+ except Exception as exc:
+ msg = _STREAM_SEEK_ERROR.format(url)
+ raise Exception(msg) from exc
+ self._bytes_downloaded = 0
+
+ self._write_to_stream(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+class RawDownload(_request_helpers.RawRequestsMixin, _download.Download):
+ """Helper to manage downloading a raw resource from a Google API.
+
+ "Slices" of the resource can be retrieved by specifying a range
+ with ``start`` and / or ``end``. However, in typical usage, neither
+ ``start`` nor ``end`` is expected to be provided.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ the downloaded resource can be written to.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, but ``end`` is provided, will download from the
+ beginning to ``end`` of the media.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, but ``start`` is provided, will download from the
+ ``start`` to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The response headers must contain
+ a checksum of the requested type. If the headers lack an
+ appropriate checksum (for instance in the case of transcoded or
+ ranged downloads where the remote service does not know the
+ correct checksum) an INFO-level log will be emitted. Supported
+ values are "md5", "crc32c" and None. The default is "md5".
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ """
+
+ def _write_to_stream(self, response):
+ """Write response body to a write-able stream.
+
+ .. note:
+
+ This method assumes that the ``_stream`` attribute is set on the
+ current download.
+
+ Args:
+ response (~requests.Response): The HTTP response object.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the download's
+ checksum doesn't agree with server-computed checksum.
+ """
+ # Retrieve the expected checksum only once for the download request,
+ # then compute and validate the checksum when the full download completes.
+ # Retried requests are range requests, and there's no way to detect
+ # data corruption for that byte range alone.
+ if self._expected_checksum is None and self._checksum_object is None:
+ # `_get_expected_checksum()` may return None even if a checksum was
+ # requested, in which case it will emit an info log _MISSING_CHECKSUM.
+ # If an invalid checksum type is specified, this will raise ValueError.
+ expected_checksum, checksum_object = _helpers._get_expected_checksum(
+ response, self._get_headers, self.media_url, checksum_type=self.checksum
+ )
+ self._expected_checksum = expected_checksum
+ self._checksum_object = checksum_object
+ else:
+ expected_checksum = self._expected_checksum
+ checksum_object = self._checksum_object
+
+ with response:
+ body_iter = response.raw.stream(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
+ )
+ for chunk in body_iter:
+ self._stream.write(chunk)
+ self._bytes_downloaded += len(chunk)
+ checksum_object.update(chunk)
+ response._content_consumed = True
+
+ # Don't validate the checksum for partial responses.
+ if (
+ expected_checksum is not None
+ and response.status_code != http.client.PARTIAL_CONTENT
+ ):
+ actual_checksum = _helpers.prepare_checksum_digest(checksum_object.digest())
+
+ if actual_checksum != expected_checksum:
+ headers = self._get_headers(response)
+ x_goog_encoding = headers.get("x-goog-stored-content-encoding")
+ x_goog_length = headers.get("x-goog-stored-content-length")
+ content_length_msg = _RESPONSE_HEADERS_INFO.format(
+ x_goog_length, x_goog_encoding, self._bytes_downloaded
+ )
+ if (
+ x_goog_length
+ and self._bytes_downloaded < int(x_goog_length)
+ and x_goog_encoding != "gzip"
+ ):
+ # The library will attempt to trigger a retry by raising a ConnectionError, if
+ # (a) bytes_downloaded is less than response header x-goog-stored-content-length, and
+ # (b) the object is not gzip-compressed when stored in Cloud Storage.
+ raise ConnectionError(content_length_msg)
+ else:
+ msg = _CHECKSUM_MISMATCH.format(
+ self.media_url,
+ expected_checksum,
+ actual_checksum,
+ checksum_type=self.checksum.upper(),
+ )
+ msg += content_length_msg
+ raise common.DataCorruption(response, msg)
+
+ def consume(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Consume the resource to be downloaded.
+
+ If a ``stream`` is attached to this download, then the downloaded
+ resource will be written to the stream.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ~google.resumable_media.common.DataCorruption: If the download's
+ checksum doesn't agree with server-computed checksum.
+ ValueError: If the current :class:`Download` has already
+ finished.
+ """
+ method, _, payload, headers = self._prepare_request()
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ request_kwargs = {
+ "data": payload,
+ "headers": headers,
+ "timeout": timeout,
+ "stream": True,
+ }
+
+ # Assign object generation if generation is specified in the media url.
+ if self._object_generation is None:
+ self._object_generation = _helpers._get_generation_from_url(self.media_url)
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ url = self.media_url
+
+ # To restart an interrupted download, read from the offset of last byte
+ # received using a range request, and set object generation query param.
+ if self._bytes_downloaded > 0:
+ _download.add_bytes_range(
+ (self.start or 0) + self._bytes_downloaded, self.end, self._headers
+ )
+ request_kwargs["headers"] = self._headers
+
+ # Set object generation query param to ensure the same object content is requested.
+ if (
+ self._object_generation is not None
+ and _helpers._get_generation_from_url(self.media_url) is None
+ ):
+ query_param = {"generation": self._object_generation}
+ url = _helpers.add_query_parameters(self.media_url, query_param)
+
+ result = transport.request(method, url, **request_kwargs)
+
+ # If a generation hasn't been specified, and this is the first response we get, let's record the
+ # generation. In future requests we'll specify the generation query param to avoid data races.
+ if self._object_generation is None:
+ self._object_generation = _helpers._parse_generation_header(
+ result, self._get_headers
+ )
+
+ self._process_response(result)
+
+ # With decompressive transcoding, GCS serves back the whole file regardless of the range request,
+ # thus we reset the stream position to the start of the stream.
+ # See: https://cloud.google.com/storage/docs/transcoding#range
+ if self._stream is not None:
+ if _helpers._is_decompressive_transcoding(result, self._get_headers):
+ try:
+ self._stream.seek(0)
+ except Exception as exc:
+ msg = _STREAM_SEEK_ERROR.format(url)
+ raise Exception(msg) from exc
+ self._bytes_downloaded = 0
+
+ self._write_to_stream(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+class ChunkedDownload(_request_helpers.RequestsMixin, _download.ChunkedDownload):
+ """Download a resource in chunks from a Google API.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each
+ request.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ will be used to concatenate chunks of the resource as they are
+ downloaded.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, defaults to ``0``.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, will download to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with each request, e.g. headers for data encryption
+ key headers.
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each request.
+
+ Raises:
+ ValueError: If ``start`` is negative.
+ """
+
+ def consume_next_chunk(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Consume the next chunk of the resource to be downloaded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ValueError: If the current download has finished.
+ """
+ method, url, payload, headers = self._prepare_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ result = transport.request(
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ timeout=timeout,
+ )
+ self._process_response(result)
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+class RawChunkedDownload(_request_helpers.RawRequestsMixin, _download.ChunkedDownload):
+ """Download a raw resource in chunks from a Google API.
+
+ Args:
+ media_url (str): The URL containing the media to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each
+ request.
+ stream (IO[bytes]): A write-able stream (i.e. file-like object) that
+ will be used to concatenate chunks of the resource as they are
+ downloaded.
+ start (int): The first byte in a range to be downloaded. If not
+ provided, defaults to ``0``.
+ end (int): The last byte in a range to be downloaded. If not
+ provided, will download to the end of the media.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with each request, e.g. headers for data encryption
+ key headers.
+
+ Attributes:
+ media_url (str): The URL containing the media to be downloaded.
+ start (Optional[int]): The first byte in a range to be downloaded.
+ end (Optional[int]): The last byte in a range to be downloaded.
+ chunk_size (int): The number of bytes to be retrieved in each request.
+
+ Raises:
+ ValueError: If ``start`` is negative.
+ """
+
+ def consume_next_chunk(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Consume the next chunk of the resource to be downloaded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ValueError: If the current download has finished.
+ """
+ method, url, payload, headers = self._prepare_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ # NOTE: We assume "payload is None" but pass it along anyway.
+ result = transport.request(
+ method,
+ url,
+ data=payload,
+ headers=headers,
+ stream=True,
+ timeout=timeout,
+ )
+ self._process_response(result)
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+def _add_decoder(response_raw, checksum):
+ """Patch the ``_decoder`` on a ``urllib3`` response.
+
+ This is so that we can intercept the compressed bytes before they are
+ decoded.
+
+ Only patches if the content encoding is ``gzip`` or ``br``.
+
+ Args:
+ response_raw (urllib3.response.HTTPResponse): The raw response for
+ an HTTP request.
+ checksum (object):
+ A checksum which will be updated with compressed bytes.
+
+ Returns:
+ object: Either the original ``checksum`` if ``_decoder`` is not
+ patched, or a ``_DoNothingHash`` if the decoder is patched, since the
+ caller will no longer need to hash to decoded bytes.
+ """
+ encoding = response_raw.headers.get("content-encoding", "").lower()
+ if encoding == "gzip":
+ response_raw._decoder = _GzipDecoder(checksum)
+ return _helpers._DoNothingHash()
+ # Only activate if brotli is installed
+ elif encoding == "br" and _BrotliDecoder: # type: ignore
+ response_raw._decoder = _BrotliDecoder(checksum)
+ return _helpers._DoNothingHash()
+ else:
+ return checksum
+
+
+class _GzipDecoder(urllib3.response.GzipDecoder):
+ """Custom subclass of ``urllib3`` decoder for ``gzip``-ed bytes.
+
+ Allows a checksum function to see the compressed bytes before they are
+ decoded. This way the checksum of the compressed value can be computed.
+
+ Args:
+ checksum (object):
+ A checksum which will be updated with compressed bytes.
+ """
+
+ def __init__(self, checksum):
+ super().__init__()
+ self._checksum = checksum
+
+ def decompress(self, data, max_length=-1):
+ """Decompress the bytes.
+
+ Args:
+ data (bytes): The compressed bytes to be decompressed.
+ max_length (int): Maximum number of bytes to return. -1 for no
+ limit. Forwarded to the underlying decoder when supported.
+
+ Returns:
+ bytes: The decompressed bytes from ``data``.
+ """
+ self._checksum.update(data)
+ try:
+ return super().decompress(data, max_length=max_length)
+ except TypeError:
+ return super().decompress(data)
+
+
+# urllib3.response.BrotliDecoder might not exist depending on whether brotli is
+# installed.
+if hasattr(urllib3.response, "BrotliDecoder"):
+
+ class _BrotliDecoder:
+ """Handler for ``brotli`` encoded bytes.
+
+ Allows a checksum function to see the compressed bytes before they are
+ decoded. This way the checksum of the compressed value can be computed.
+
+ Because BrotliDecoder's decompress method is dynamically created in
+ urllib3, a subclass is not practical. Instead, this class creates a
+ captive urllib3.requests.BrotliDecoder instance and acts as a proxy.
+
+ Args:
+ checksum (object):
+ A checksum which will be updated with compressed bytes.
+ """
+
+ def __init__(self, checksum):
+ self._decoder = urllib3.response.BrotliDecoder()
+ self._checksum = checksum
+
+ def decompress(self, data, max_length=-1):
+ """Decompress the bytes.
+
+ Args:
+ data (bytes): The compressed bytes to be decompressed.
+ max_length (int): Maximum number of bytes to return. -1 for no
+ limit. Forwarded to the underlying decoder when supported.
+
+ Returns:
+ bytes: The decompressed bytes from ``data``.
+ """
+ self._checksum.update(data)
+ try:
+ return self._decoder.decompress(data, max_length=max_length)
+ except TypeError:
+ return self._decoder.decompress(data)
+
+ @property
+ def has_unconsumed_tail(self):
+ try:
+ return self._decoder.has_unconsumed_tail
+ except AttributeError:
+ return False
+
+ def flush(self):
+ return self._decoder.flush()
+
+else: # pragma: NO COVER
+ _BrotliDecoder = None # type: ignore # pragma: NO COVER
diff --git a/packages/google-resumable-media/google/resumable_media/requests/upload.py b/packages/google-resumable-media/google/resumable_media/requests/upload.py
new file mode 100644
index 000000000000..3233517d0930
--- /dev/null
+++ b/packages/google-resumable-media/google/resumable_media/requests/upload.py
@@ -0,0 +1,761 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Support for resumable uploads.
+
+Also supported here are simple (media) uploads and multipart
+uploads that contain both metadata and a small file as payload.
+"""
+
+from google.resumable_media import _upload
+from google.resumable_media.requests import _request_helpers
+
+
+class SimpleUpload(_request_helpers.RequestsMixin, _upload.SimpleUpload):
+ """Upload a resource to a Google API.
+
+ A **simple** media upload sends no metadata and completes the upload
+ in a single request.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ def transmit(
+ self,
+ transport,
+ data,
+ content_type,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Transmit the resource to be uploaded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ data (bytes): The resource content to be uploaded.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_request(data, content_type)
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+class MultipartUpload(_request_helpers.RequestsMixin, _upload.MultipartUpload):
+ """Upload a resource with metadata to a Google API.
+
+ A **multipart** upload sends both metadata and the resource in a single
+ (multipart) request.
+
+ Args:
+ upload_url (str): The URL where the content will be uploaded.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the request, e.g. headers for encrypted data.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. The request metadata will be amended
+ to include the computed value. Using this option will override a
+ manually-set checksum value. Supported values are "md5",
+ "crc32c" and None. The default is None.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ """
+
+ def transmit(
+ self,
+ transport,
+ data,
+ metadata,
+ content_type,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Transmit the resource to be uploaded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ data (bytes): The resource content to be uploaded.
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_request(
+ data, metadata, content_type
+ )
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+class ResumableUpload(_request_helpers.RequestsMixin, _upload.ResumableUpload):
+ """Initiate and fulfill a resumable upload to a Google API.
+
+ A **resumable** upload sends an initial request with the resource metadata
+ and then gets assigned an upload ID / upload URL to send bytes to.
+ Using the upload URL, the upload is then done in chunks (determined by
+ the user) until all bytes have been uploaded.
+
+ When constructing a resumable upload, only the resumable upload URL and
+ the chunk size are required:
+
+ .. testsetup:: resumable-constructor
+
+ bucket = 'bucket-foo'
+
+ .. doctest:: resumable-constructor
+
+ >>> from google.resumable_media.requests import ResumableUpload
+ >>>
+ >>> url_template = (
+ ... 'https://www.googleapis.com/upload/storage/v1/b/{bucket}/o?'
+ ... 'uploadType=resumable')
+ >>> upload_url = url_template.format(bucket=bucket)
+ >>>
+ >>> chunk_size = 3 * 1024 * 1024 # 3MB
+ >>> upload = ResumableUpload(upload_url, chunk_size)
+
+ When initiating an upload (via :meth:`initiate`), the caller is expected
+ to pass the resource being uploaded as a file-like ``stream``. If the size
+ of the resource is explicitly known, it can be passed in directly:
+
+ .. testsetup:: resumable-explicit-size
+
+ import os
+ import tempfile
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import ResumableUpload
+
+ upload_url = 'http://test.invalid'
+ chunk_size = 3 * 1024 * 1024 # 3MB
+ upload = ResumableUpload(upload_url, chunk_size)
+
+ file_desc, filename = tempfile.mkstemp()
+ os.close(file_desc)
+
+ data = b'some bytes!'
+ with open(filename, 'wb') as file_obj:
+ file_obj.write(data)
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response._content = b''
+ resumable_url = 'http://test.invalid?upload_id=7up'
+ fake_response.headers['location'] = resumable_url
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+ .. doctest:: resumable-explicit-size
+
+ >>> import os
+ >>>
+ >>> upload.total_bytes is None
+ True
+ >>>
+ >>> stream = open(filename, 'rb')
+ >>> total_bytes = os.path.getsize(filename)
+ >>> metadata = {'name': filename}
+ >>> response = upload.initiate(
+ ... transport, stream, metadata, 'text/plain',
+ ... total_bytes=total_bytes)
+ >>> response
+
+ >>>
+ >>> upload.total_bytes == total_bytes
+ True
+
+ .. testcleanup:: resumable-explicit-size
+
+ os.remove(filename)
+
+ If the stream is in a "final" state (i.e. it won't have any more bytes
+ written to it), the total number of bytes can be determined implicitly
+ from the ``stream`` itself:
+
+ .. testsetup:: resumable-implicit-size
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import ResumableUpload
+
+ upload_url = 'http://test.invalid'
+ chunk_size = 3 * 1024 * 1024 # 3MB
+ upload = ResumableUpload(upload_url, chunk_size)
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response._content = b''
+ resumable_url = 'http://test.invalid?upload_id=7up'
+ fake_response.headers['location'] = resumable_url
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+ data = b'some MOAR bytes!'
+ metadata = {'name': 'some-file.jpg'}
+ content_type = 'image/jpeg'
+
+ .. doctest:: resumable-implicit-size
+
+ >>> stream = io.BytesIO(data)
+ >>> response = upload.initiate(
+ ... transport, stream, metadata, content_type)
+ >>>
+ >>> upload.total_bytes == len(data)
+ True
+
+ If the size of the resource is **unknown** when the upload is initiated,
+ the ``stream_final`` argument can be used. This might occur if the
+ resource is being dynamically created on the client (e.g. application
+ logs). To use this argument:
+
+ .. testsetup:: resumable-unknown-size
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ from google.resumable_media.requests import ResumableUpload
+
+ upload_url = 'http://test.invalid'
+ chunk_size = 3 * 1024 * 1024 # 3MB
+ upload = ResumableUpload(upload_url, chunk_size)
+
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.OK)
+ fake_response._content = b''
+ resumable_url = 'http://test.invalid?upload_id=7up'
+ fake_response.headers['location'] = resumable_url
+
+ post_method = mock.Mock(return_value=fake_response, spec=[])
+ transport = mock.Mock(request=post_method, spec=['request'])
+
+ metadata = {'name': 'some-file.jpg'}
+ content_type = 'application/octet-stream'
+
+ stream = io.BytesIO(b'data')
+
+ .. doctest:: resumable-unknown-size
+
+ >>> response = upload.initiate(
+ ... transport, stream, metadata, content_type,
+ ... stream_final=False)
+ >>>
+ >>> upload.total_bytes is None
+ True
+
+ Args:
+ upload_url (str): The URL where the resumable upload will be initiated.
+ chunk_size (int): The size of each chunk used to upload the resource.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the :meth:`initiate` request, e.g. headers for
+ encrypted data. These **will not** be sent with
+ :meth:`transmit_next_chunk` or :meth:`recover` requests.
+ checksum Optional([str]): The type of checksum to compute to verify
+ the integrity of the object. After the upload is complete, the
+ server-computed checksum of the resulting object will be checked
+ and google.resumable_media.common.DataCorruption will be raised on
+ a mismatch. The corrupted file will not be deleted from the remote
+ host automatically. Supported values are "md5", "crc32c" and None.
+ The default is None.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+
+ Raises:
+ ValueError: If ``chunk_size`` is not a multiple of
+ :data:`.UPLOAD_CHUNK_SIZE`.
+ """
+
+ def initiate(
+ self,
+ transport,
+ stream,
+ metadata,
+ content_type,
+ total_bytes=None,
+ stream_final=True,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Initiate a resumable upload.
+
+ By default, this method assumes your ``stream`` is in a "final"
+ state ready to transmit. However, ``stream_final=False`` can be used
+ to indicate that the size of the resource is not known. This can happen
+ if bytes are being dynamically fed into ``stream``, e.g. if the stream
+ is attached to application logs.
+
+ If ``stream_final=False`` is used, :attr:`chunk_size` bytes will be
+ read from the stream every time :meth:`transmit_next_chunk` is called.
+ If one of those reads produces strictly fewer bites than the chunk
+ size, the upload will be concluded.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ stream (IO[bytes]): The stream (i.e. file-like object) that will
+ be uploaded. The stream **must** be at the beginning (i.e.
+ ``stream.tell() == 0``).
+ metadata (Mapping[str, str]): The resource metadata, such as an
+ ACL list.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ total_bytes (Optional[int]): The total number of bytes to be
+ uploaded. If specified, the upload size **will not** be
+ determined from the stream (even if ``stream_final=True``).
+ stream_final (Optional[bool]): Indicates if the ``stream`` is
+ "final" (i.e. no more bytes will be added to it). In this case
+ we determine the upload size from the size of the stream. If
+ ``total_bytes`` is passed, this argument will be ignored.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_initiate_request(
+ stream,
+ metadata,
+ content_type,
+ total_bytes=total_bytes,
+ stream_final=stream_final,
+ )
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_initiate_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+ def transmit_next_chunk(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Transmit the next chunk of the resource to be uploaded.
+
+ If the current upload was initiated with ``stream_final=False``,
+ this method will dynamically determine if the upload has completed.
+ The upload will be considered complete if the stream produces
+ fewer than :attr:`chunk_size` bytes when a chunk is read from it.
+
+ In the case of failure, an exception is thrown that preserves the
+ failed response:
+
+ .. testsetup:: bad-response
+
+ import io
+
+ import mock
+ import requests
+ import http.client
+
+ from google import resumable_media
+ import google.resumable_media.requests.upload as upload_mod
+
+ transport = mock.Mock(spec=['request'])
+ fake_response = requests.Response()
+ fake_response.status_code = int(http.client.BAD_REQUEST)
+ transport.request.return_value = fake_response
+
+ upload_url = 'http://test.invalid'
+ upload = upload_mod.ResumableUpload(
+ upload_url, resumable_media.UPLOAD_CHUNK_SIZE)
+ # Fake that the upload has been initiate()-d
+ data = b'data is here'
+ upload._stream = io.BytesIO(data)
+ upload._total_bytes = len(data)
+ upload._resumable_url = 'http://test.invalid?upload_id=nope'
+
+ .. doctest:: bad-response
+ :options: +NORMALIZE_WHITESPACE
+
+ >>> error = None
+ >>> try:
+ ... upload.transmit_next_chunk(transport)
+ ... except resumable_media.InvalidResponse as caught_exc:
+ ... error = caught_exc
+ ...
+ >>> error
+ InvalidResponse('Request failed with status code', 400,
+ 'Expected one of', , )
+ >>> error.response
+
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+
+ Raises:
+ ~google.resumable_media.common.InvalidResponse: If the status
+ code is not 200 or http.client.PERMANENT_REDIRECT.
+ ~google.resumable_media.common.DataCorruption: If this is the final
+ chunk, a checksum validation was requested, and the checksum
+ does not match or is not available.
+ """
+ method, url, payload, headers = self._prepare_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_resumable_response(result, len(payload))
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+ def recover(self, transport):
+ """Recover from a failure and check the status of the current upload.
+
+ This will verify the progress with the server and make sure the
+ current upload is in a valid state before :meth:`transmit_next_chunk`
+ can be used again. See https://cloud.google.com/storage/docs/performing-resumable-uploads#status-check
+ for more information.
+
+ This method can be used when a :class:`ResumableUpload` is in an
+ :attr:`~ResumableUpload.invalid` state due to a request failure.
+
+ Args:
+ transport (~requests.Session): A ``requests`` object which can
+ make authenticated requests.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ timeout = (
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ )
+
+ method, url, payload, headers = self._prepare_recover_request()
+ # NOTE: We assume "payload is None" but pass it along anyway.
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_recover_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+class XMLMPUContainer(_request_helpers.RequestsMixin, _upload.XMLMPUContainer):
+ """Initiate and close an upload using the XML MPU API.
+
+ An XML MPU sends an initial request and then receives an upload ID.
+ Using the upload ID, the upload is then done in numbered parts and the
+ parts can be uploaded concurrently.
+
+ In order to avoid concurrency issues with this container object, the
+ uploading of individual parts is handled separately, by XMLMPUPart objects
+ spawned from this container class. The XMLMPUPart objects are not
+ necessarily in the same process as the container, so they do not update the
+ container automatically.
+
+ MPUs are sometimes referred to as "Multipart Uploads", which is ambiguous
+ given the JSON multipart upload, so the abbreviation "MPU" will be used
+ throughout.
+
+ See: https://cloud.google.com/storage/docs/multipart-uploads
+
+ Args:
+ upload_url (str): The URL of the object (without query parameters). The
+ initiate, PUT, and finalization requests will all use this URL, with
+ varying query parameters.
+ headers (Optional[Mapping[str, str]]): Extra headers that should
+ be sent with the :meth:`initiate` request, e.g. headers for
+ encrypted data. These headers will be propagated to individual
+ XMLMPUPart objects spawned from this container as well.
+
+ Attributes:
+ upload_url (str): The URL where the content will be uploaded.
+ upload_id (Optional(int)): The ID of the upload from the initialization
+ response.
+ """
+
+ def initiate(
+ self,
+ transport,
+ content_type,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Initiate an MPU and record the upload ID.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ content_type (str): The content type of the resource, e.g. a JPEG
+ image has content type ``image/jpeg``.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+
+ method, url, payload, headers = self._prepare_initiate_request(
+ content_type,
+ )
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_initiate_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+ def finalize(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Finalize an MPU request with all the parts.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_finalize_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_finalize_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+ def cancel(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Cancel an MPU request and permanently delete any uploaded parts.
+
+ This cannot be undone.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_cancel_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_cancel_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
+
+
+class XMLMPUPart(_request_helpers.RequestsMixin, _upload.XMLMPUPart):
+ def upload(
+ self,
+ transport,
+ timeout=(
+ _request_helpers._DEFAULT_CONNECT_TIMEOUT,
+ _request_helpers._DEFAULT_READ_TIMEOUT,
+ ),
+ ):
+ """Upload the part.
+
+ Args:
+ transport (object): An object which can make authenticated
+ requests.
+ timeout (Optional[Union[float, Tuple[float, float]]]):
+ The number of seconds to wait for the server response.
+ Depending on the retry strategy, a request may be repeated
+ several times using the same timeout each time.
+
+ Can also be passed as a tuple (connect_timeout, read_timeout).
+ See :meth:`requests.Session.request` documentation for details.
+
+ Returns:
+ ~requests.Response: The HTTP response returned by ``transport``.
+ """
+ method, url, payload, headers = self._prepare_upload_request()
+
+ # Wrap the request business logic in a function to be retried.
+ def retriable_request():
+ result = transport.request(
+ method, url, data=payload, headers=headers, timeout=timeout
+ )
+
+ self._process_upload_response(result)
+
+ return result
+
+ return _request_helpers.wait_and_retry(
+ retriable_request, self._get_status_code, self._retry_strategy
+ )
diff --git a/packages/google-resumable-media/mypy.ini b/packages/google-resumable-media/mypy.ini
new file mode 100644
index 000000000000..4505b485436b
--- /dev/null
+++ b/packages/google-resumable-media/mypy.ini
@@ -0,0 +1,3 @@
+[mypy]
+python_version = 3.6
+namespace_packages = True
diff --git a/packages/google-resumable-media/noxfile.py b/packages/google-resumable-media/noxfile.py
new file mode 100644
index 000000000000..0bf3e504fdc5
--- /dev/null
+++ b/packages/google-resumable-media/noxfile.py
@@ -0,0 +1,319 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import os
+import pathlib
+import shutil
+
+import nox
+
+CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
+
+SYSTEM_TEST_ENV_VARS = ("GOOGLE_APPLICATION_CREDENTIALS",)
+RUFF_VERSION = "ruff==0.14.14"
+
+DEFAULT_PYTHON_VERSION = "3.14"
+UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+SYSTEM_TEST_PYTHON_VERSIONS = UNIT_TEST_PYTHON_VERSIONS
+
+# Error if a python version is missing
+nox.options.error_on_missing_interpreters = True
+
+nox.options.sessions = [
+ "system",
+ "blacken",
+ "mypy",
+ "doctest",
+]
+
+
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
+def unit(session):
+ """Run the unit test suite."""
+
+ constraints_path = str(
+ CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+ )
+
+ # Install all test dependencies, then install this package in-place.
+ session.install("mock", "pytest", "pytest-cov", "pytest-asyncio<=0.14.0", "brotli")
+ session.install("-e", ".[requests,aiohttp]", "-c", constraints_path)
+
+ # Run py.test against the unit tests.
+ # NOTE: We don't require 100% line coverage for unit test runs since
+ # some have branches that are Py2/Py3 specific.
+ line_coverage = "--cov-fail-under=0"
+ session.run(
+ "py.test",
+ "--cov=google.resumable_media",
+ "--cov=google._async_resumable_media",
+ "--cov=tests.unit",
+ "--cov=tests_async.unit",
+ "--cov-append",
+ "--cov-config=.coveragerc",
+ "--cov-report=",
+ line_coverage,
+ os.path.join("tests", "unit"),
+ os.path.join("tests_async", "unit"),
+ *session.posargs
+ )
+
+
+@nox.session(python="3.10")
+def docs(session):
+ """Build the docs for this library."""
+
+ session.install("-e", ".")
+ session.install(
+ # We need to pin to specific versions of the `sphinxcontrib-*` packages
+ # which still support sphinx 4.x.
+ # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
+ # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
+ "sphinxcontrib-applehelp==1.0.4",
+ "sphinxcontrib-devhelp==1.0.2",
+ "sphinxcontrib-htmlhelp==2.0.1",
+ "sphinxcontrib-qthelp==1.0.3",
+ "sphinxcontrib-serializinghtml==1.1.5",
+ "sphinx==4.5.0",
+ "alabaster",
+ "recommonmark",
+ )
+
+ shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
+ session.run(
+ "sphinx-build",
+ "-W", # warnings as errors
+ "-T", # show full traceback on exception
+ "-N", # no colors
+ "-b",
+ "html",
+ "-d",
+ os.path.join("docs", "_build", "doctrees", ""),
+ os.path.join("docs", ""),
+ os.path.join("docs", "_build", "html", ""),
+ )
+
+@nox.session(python="3.10")
+def docfx(session):
+ """Build the docfx yaml files for this library."""
+
+ session.install("-e", ".")
+ session.install(
+ # We need to pin to specific versions of the `sphinxcontrib-*` packages
+ # which still support sphinx 4.x.
+ # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344
+ # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345.
+ "sphinxcontrib-applehelp==1.0.4",
+ "sphinxcontrib-devhelp==1.0.2",
+ "sphinxcontrib-htmlhelp==2.0.1",
+ "sphinxcontrib-qthelp==1.0.3",
+ "sphinxcontrib-serializinghtml==1.1.5",
+ "gcp-sphinx-docfx-yaml",
+ "alabaster",
+ "recommonmark",
+ )
+
+ shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True)
+ session.run(
+ "sphinx-build",
+ "-T", # show full traceback on exception
+ "-N", # no colors
+ "-D",
+ (
+ "extensions=sphinx.ext.autodoc,"
+ "sphinx.ext.autosummary,"
+ "docfx_yaml.extension,"
+ "sphinx.ext.intersphinx,"
+ "sphinx.ext.coverage,"
+ "sphinx.ext.napoleon,"
+ "sphinx.ext.todo,"
+ "sphinx.ext.viewcode,"
+ "recommonmark"
+ ),
+ "-b",
+ "html",
+ "-d",
+ os.path.join("docs", "_build", "doctrees", ""),
+ os.path.join("docs", ""),
+ os.path.join("docs", "_build", "html", ""),
+ )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def doctest(session):
+ """Run the doctests."""
+ session.install("-e", ".[requests,aiohttp]")
+ session.install("sphinx==4.0.1", "alabaster", "recommonmark")
+ session.install(
+ "sphinxcontrib-applehelp==1.0.4",
+ "sphinxcontrib-devhelp==1.0.2",
+ "sphinxcontrib-htmlhelp==2.0.1",
+ "sphinxcontrib-qthelp==1.0.3",
+ "sphinxcontrib-serializinghtml==1.1.5",
+ "sphinx==4.5.0",
+ "sphinx_rtd_theme",
+ "sphinx-docstring-typing >= 0.0.3",
+ "mock",
+ )
+
+ # Run the doctests with Sphinx.
+ session.run(
+ "sphinx-build",
+ "-W",
+ "-b",
+ "doctest",
+ "-d",
+ os.path.join("docs", "_build", "doctrees"),
+ os.path.join("docs", ""),
+ os.path.join("docs", "_build", "doctest"),
+ )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def lint(session):
+ """Run flake8.
+
+ Returns a failure if flake8 finds linting errors or sufficiently
+ serious code quality issues.
+ """
+ session.install("flake8", RUFF_VERSION)
+ session.install("-e", ".")
+ session.run(
+ "flake8",
+ os.path.join("google", "resumable_media"),
+ "tests",
+ os.path.join("google", "_async_resumable_media"),
+ "tests_async",
+ )
+
+ # 2. Check formatting
+ session.run(
+ "ruff",
+ "format",
+ "--check",
+ f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}",
+ "--line-length=88",
+ os.path.join("google", "resumable_media"),
+ "tests",
+ os.path.join("google", "_async_resumable_media"),
+ "tests_async",
+ )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def lint_setup_py(session):
+ """Verify that setup.py is valid (including RST check)."""
+ session.install("setuptools", "docutils", "Pygments")
+ session.run("python", "setup.py", "check", "--restructuredtext", "--strict")
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def blacken(session):
+ """(Deprecated) Legacy session. Please use 'nox -s format'."""
+ session.log(
+ "WARNING: The 'blacken' session is deprecated and will be removed in a future release. Please use 'nox -s format' in the future."
+ )
+
+ # Just run the ruff formatter (keeping legacy behavior of only formatting, not sorting imports)
+ session.install(RUFF_VERSION)
+ session.run(
+ "ruff",
+ "format",
+ f"--target-version=py{UNIT_TEST_PYTHON_VERSIONS[0].replace('.', '')}",
+ "--line-length=88",
+ os.path.join("google", "resumable_media"),
+ "tests",
+ os.path.join("google", "_async_resumable_media"),
+ "tests_async",
+ )
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def mypy(session):
+ """Verify type hints are mypy compatible."""
+ session.install("-e", ".")
+ session.install(
+ "mypy",
+ "types-setuptools",
+ "types-requests",
+ "types-mock",
+ )
+ session.run("mypy", "-p", "google", "-p", "tests", "-p", "tests_async")
+
+
+@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
+def system(session):
+ """Run the system test suite."""
+
+ constraints_path = str(
+ CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
+ )
+
+ # Environment check: environment variables are set.
+ missing = []
+ for env_var in SYSTEM_TEST_ENV_VARS:
+ if env_var not in os.environ:
+ missing.append(env_var)
+
+ # Only run system tests if the environment variables are set.
+ if missing:
+ all_vars = ", ".join(missing)
+ msg = "Environment variable(s) unset: {}".format(all_vars)
+ session.skip(msg)
+
+ # Install all test dependencies, then install this package into the
+ # virtualenv's dist-packages.
+ session.install("mock", "pytest", "google-cloud-testutils", "brotli")
+ session.install("-e", ".[requests,aiohttp]", "-c", constraints_path)
+
+ # Run py.test against the async system tests.
+ if session.python.startswith("3"):
+ session.install("pytest-asyncio<=0.14.0")
+ session.run(
+ "py.test", "-s", os.path.join("tests_async", "system"), *session.posargs
+ )
+
+ # Run py.test against the system tests.
+ session.run("py.test", "-s", os.path.join("tests", "system"), *session.posargs)
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def cover(session):
+ """Run the final coverage report.
+
+ This outputs the coverage report aggregating coverage from the unit
+ test runs (not system test runs), and then erases coverage data.
+ """
+ session.install("coverage", "pytest-cov")
+ session.run("coverage", "report", "--show-missing", "--fail-under=100")
+ session.run("coverage", "erase")
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def prerelease_deps(session):
+ # TODO(https://github.com/googleapis/google-cloud-python/issues/16014):
+ # Resolve the linked bug once prerelease_deps and core_deps_from_source
+ # are implemented for this package.
+ if session.python == DEFAULT_PYTHON_VERSION:
+ session.skip(f"Skipping prerelease_deps for {DEFAULT_PYTHON_VERSION} until a future release.")
+
+
+@nox.session(python=DEFAULT_PYTHON_VERSION)
+def core_deps_from_source(session):
+ ## TODO(https://github.com/googleapis/google-cloud-python/issues/16014):
+ # Resolve the linked bug once prerelease_deps and core_deps_from_source
+ # are implemented for this package.
+ if session.python == DEFAULT_PYTHON_VERSION:
+ session.skip(f"Skipping core_deps_from_source for {DEFAULT_PYTHON_VERSION} until a future release.")
diff --git a/packages/google-resumable-media/setup.cfg b/packages/google-resumable-media/setup.cfg
new file mode 100644
index 000000000000..add5a13f9640
--- /dev/null
+++ b/packages/google-resumable-media/setup.cfg
@@ -0,0 +1,2 @@
+[tool:pytest]
+addopts = --tb=native
diff --git a/packages/google-resumable-media/setup.py b/packages/google-resumable-media/setup.py
new file mode 100644
index 000000000000..7c09ac8a5a17
--- /dev/null
+++ b/packages/google-resumable-media/setup.py
@@ -0,0 +1,71 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import setuptools
+
+
+PACKAGE_ROOT = os.path.abspath(os.path.dirname(__file__))
+
+with open(os.path.join(PACKAGE_ROOT, 'README.rst')) as file_obj:
+ README = file_obj.read()
+
+
+REQUIREMENTS = [
+ 'google-crc32c >= 1.0.0, < 2.0.0',
+]
+EXTRAS_REQUIRE = {
+ 'requests': [
+ 'requests >= 2.18.0, < 3.0.0',
+ ],
+ 'aiohttp': ['aiohttp >= 3.6.2, < 4.0.0', 'google-auth >= 1.22.0, < 2.0.0']
+}
+
+setuptools.setup(
+ name='google-resumable-media',
+ version = "2.8.0",
+ description='Utilities for Google Media Downloads and Resumable Uploads',
+ author='Google Cloud Platform',
+ author_email='googleapis-publisher@google.com',
+ long_description=README,
+ scripts=[],
+ url='https://github.com/googleapis/google-resumable-media-python',
+ packages=setuptools.find_namespace_packages(
+ exclude=("tests*", "docs*")
+ ),
+ license='Apache 2.0',
+ platforms='Posix; MacOS X; Windows',
+ include_package_data=True,
+ zip_safe=False,
+ install_requires=REQUIREMENTS,
+ extras_require=EXTRAS_REQUIRE,
+ python_requires='>= 3.7',
+ classifiers=[
+ 'Development Status :: 5 - Production/Stable',
+ 'Intended Audience :: Developers',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python :: 3',
+ 'Programming Language :: Python :: 3.7',
+ 'Programming Language :: Python :: 3.8',
+ 'Programming Language :: Python :: 3.9',
+ 'Programming Language :: Python :: 3.10',
+ 'Programming Language :: Python :: 3.11',
+ 'Programming Language :: Python :: 3.12',
+ 'Programming Language :: Python :: 3.13',
+ 'Programming Language :: Python :: 3.14',
+ 'Topic :: Internet',
+ ],
+)
diff --git a/packages/google-resumable-media/testing/constraints-3.10.txt b/packages/google-resumable-media/testing/constraints-3.10.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-resumable-media/testing/constraints-3.11.txt b/packages/google-resumable-media/testing/constraints-3.11.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-resumable-media/testing/constraints-3.12.txt b/packages/google-resumable-media/testing/constraints-3.12.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-resumable-media/testing/constraints-3.13.txt b/packages/google-resumable-media/testing/constraints-3.13.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-resumable-media/testing/constraints-3.14.txt b/packages/google-resumable-media/testing/constraints-3.14.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-resumable-media/testing/constraints-3.7.txt b/packages/google-resumable-media/testing/constraints-3.7.txt
new file mode 100644
index 000000000000..fa3aa7d0a1cc
--- /dev/null
+++ b/packages/google-resumable-media/testing/constraints-3.7.txt
@@ -0,0 +1,4 @@
+crcmod==1.7
+google-crc32c==1.0
+aiohttp==3.6.2
+requests==2.23.0
diff --git a/packages/google-resumable-media/testing/constraints-3.8.txt b/packages/google-resumable-media/testing/constraints-3.8.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-resumable-media/testing/constraints-3.9.txt b/packages/google-resumable-media/testing/constraints-3.9.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-resumable-media/tests/__init__.py b/packages/google-resumable-media/tests/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests/data/brotli.txt b/packages/google-resumable-media/tests/data/brotli.txt
new file mode 100644
index 000000000000..da07c51074cb
--- /dev/null
+++ b/packages/google-resumable-media/tests/data/brotli.txt
@@ -0,0 +1,64 @@
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
diff --git a/packages/google-resumable-media/tests/data/brotli.txt.br b/packages/google-resumable-media/tests/data/brotli.txt.br
new file mode 100644
index 000000000000..84828432cc5c
Binary files /dev/null and b/packages/google-resumable-media/tests/data/brotli.txt.br differ
diff --git a/packages/google-resumable-media/tests/data/favicon.ico b/packages/google-resumable-media/tests/data/favicon.ico
new file mode 100644
index 000000000000..e9c59160aa3c
Binary files /dev/null and b/packages/google-resumable-media/tests/data/favicon.ico differ
diff --git a/packages/google-resumable-media/tests/data/file.txt b/packages/google-resumable-media/tests/data/file.txt
new file mode 100644
index 000000000000..da07c51074cb
--- /dev/null
+++ b/packages/google-resumable-media/tests/data/file.txt
@@ -0,0 +1,64 @@
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
diff --git a/packages/google-resumable-media/tests/data/gzipped.txt b/packages/google-resumable-media/tests/data/gzipped.txt
new file mode 100644
index 000000000000..da07c51074cb
--- /dev/null
+++ b/packages/google-resumable-media/tests/data/gzipped.txt
@@ -0,0 +1,64 @@
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
diff --git a/packages/google-resumable-media/tests/data/gzipped.txt.gz b/packages/google-resumable-media/tests/data/gzipped.txt.gz
new file mode 100644
index 000000000000..83e9f396c3c2
Binary files /dev/null and b/packages/google-resumable-media/tests/data/gzipped.txt.gz differ
diff --git a/packages/google-resumable-media/tests/data/image1.jpg b/packages/google-resumable-media/tests/data/image1.jpg
new file mode 100644
index 000000000000..e70137b82794
Binary files /dev/null and b/packages/google-resumable-media/tests/data/image1.jpg differ
diff --git a/packages/google-resumable-media/tests/data/image2.jpg b/packages/google-resumable-media/tests/data/image2.jpg
new file mode 100644
index 000000000000..c3969530e139
Binary files /dev/null and b/packages/google-resumable-media/tests/data/image2.jpg differ
diff --git a/packages/google-resumable-media/tests/system/__init__.py b/packages/google-resumable-media/tests/system/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests/system/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests/system/credentials.json.enc b/packages/google-resumable-media/tests/system/credentials.json.enc
new file mode 100644
index 000000000000..19e26ade73f1
--- /dev/null
+++ b/packages/google-resumable-media/tests/system/credentials.json.enc
@@ -0,0 +1,52 @@
+U2FsdGVkX1+wqu1+eVu6OPbPoE0lzIp3B11p8Rdbha1ukxXcsskegJdBjcUqQOav
+W2N3vhA7YfXW/F3T+tZMYYWk5a0vAjxLov3MgFfhvGPK0UzDwKNIXRgxhcLjcSeQ
+ZmSN2kqpmSSKEPLxP0B6r50nAG6r8NYbZWs02lH2e3NGbsoGgP5PQV2oP/ZVYkET
+qABgSd+xkOjE/7664QRfs/5Jl3Pl045Mzl87l1kN6oeoFpxeFqGWOR4WNflauS3s
+96SKsbrCQ4aF/9n9hCz31J9cJosu54eTB9s0fKBkDx7xmouwT3Cqv2KGwJPUCRHk
+3a+3ijxhNz65dYCRp20dUpJuudFQvMpsptn7oAFtNQhvcFrpjnyBn3ODr9JhLBEy
+PTdJbv06ufb+SH9YNMpH3nTYCkS7ZgrnzhteFJtoMzX6sAYiMUmIZtGY7J8MaSE0
+AYqTO/EGkzzSw33o2nNGcg0lsW1tdmY5GKuJ3jlc1Hi6RHpmgbdv+0dAYi734sYs
++0wE18QMe4/RIOCBslMAWvlo9LX9QDLkolToToQ+HN/kJNQOumkxwcjBV3piiJQH
+LaX9bI6lnqkoMl/2GvuR+oQTfzQxjGKdenLWZO2ODH2rr90hXi9vlXjdpDGreMGy
+Mv4lcwmw3Pd1JreKJtdc2ObDrU/o7wDJe4txNCGwCSAZacI+5c/27mT1yOfgE/EK
+Q3LHjqZhFlLI4K0KqH+dyQutL7b1uPtQpeWAVAt/yHs7nNWF62UAdVR+hZyko2Dy
+HWoYtJDMazfpS98c8VWi0FyGfYVESedWvBCLHch4wWqaccY0HWk9sehyC4XrPX8v
+OMw6J1va3vprzCQte56fXNzzpU6f0XeT3OGj5RCN/POMnN+cjyuwqFOsWNCfpXaV
+lhNj3zg+fMk4mM+wa2KdUk6xa0vj7YblgJ5uvZ3lG81ydZCRoFWqaO6497lnj8NV
+SEDqDdJ+/dw+Sf2ur3hyJ9DW0JD8QJkSwfLrqT51eoOqTfFFGdwy2iuXP426l/NH
+mkyusp8UZNPaKZSF9jC8++18fC2Nbbd+dTIn6XWdZKKRZLZ/hca8QP0QesrtYo36
+6kx8Kl3nAbgOk9wFFsZdkUyOy3iRxkBF0qoaH1kPzyxIpNeeIg5cBPWLwN5FVBdd
+eBy8R4i4y/W8yhib34vcOliP0IfAB/VvXJRMUCc1bENfZskMb4mvtsYblyf68Fne
+OjtcSKV2drO+mRmH1H2sPH/yE2yVDivhY5FJxDRFMnS9HXDMpGoukirMLgCjnSre
+ZXMVaDzkRw1RtsOms+F7EVJb5v/HKu6I34YNJDlAFy6AASmz+H0EXBDK4mma8GSu
+BOgPY3PbF8R+KnzKsOVbaOon90dGclnUNlqnVvsnNeWWKJmL7rCPkMHfb5dBhw60
+j9oLmu74+xmuf9aqzSvrcaHV9u+zf2eCsdQJhttaDYFAKg1q43fhZYHIaURidoD+
+UTxn0AVygiKkTwTFQl1+taDiRffOtNvumSLZG9n8cimoBvzKle3H9tv43uyO6muG
+ty0m8Pyk5LyLE9DaDQwxq+++8g7boXQe7jCtAIMxRveIdwWPI/XHbyZ3I4uTG65F
+RV5K8Q34VVjagdPMNq0ijo73iYy5RH18MSQc8eG3UtqVvr/QeSdPEb8N6o+OwEG8
+VuAFbKPHMfQrjwGCtr0YvHTmvZPlFef+J3iH6WPfFFbe5ZS8XQUoR1dZHX9BXIXK
+Om/itKUoHvAuYIqjTboqK181OVr/9a2FipXxbenXYiWXRtLGpHeetZbKRhxwWe0h
+kDdDL/XglsRNasfLz4c9AyGzJJi7J9Pr7uBSX9QFHLeGQP6jfHrEqBkiGEUP9iQr
+11wabtNouC+1tT0erBAm/KEps81l76NZ7OxqOM8mLrdAE8RO/ypZTqZW4saQnry/
+iUGhwEnRNZpEh8xiYSZ8JgUTbbKo4+FXZxUwV1DBQ7oroPrduaukd68m4E6Tqsx+
+lTl25hLhNTEJCYQ0hg2CeZdSpOPGgpn+zhLDvlQ0lPZDCByh9xCepAq/oUArddln
+vobPdBRVW27gYntAYMlFbc1hSN/LKoZOYq6jBNAPykiv5tTWNV71HUE7b1nRfo27
+aGf3Ptzu7GRXVLom+WKxswUqzkWC8afvrNnZ040wiLQnWzn2yxytipUg3UxIvP+U
+klWj8Tt1wBmG/JGLEThwcjPTOGvDkocQAAImlV3diiqwTHlj+pLZVRtJA4SOQxI8
+ChFi73B8gPOexfqYPUFdB90FJWsxTQGZaucyuNTqFMuJ9eEDP5WmK4lcJuKFTCGT
+M4VYd9j4JlxRRQxKkMhfoXeUsW3TH6uAmKxN79AiYnOh6QUIv+PP+yt9WwQhNqkb
+7otLl0AKdMBizxyq6AExlw/VmdYDJxcZ4Y/P+M85Ae5e+Lz/XjWHLnjP1BPI6C+n
+A/RbICOd/W/wf6ZOZlVBW1wePv0M5jWDGL086lHVrgBnzdWrQTHhzG43v1IaN/vK
+EVZfvkqTe5AWNoK1Da/zEafWf0jzc4cS0grCA9KJ0nHwRYYEG0YQAGqY12PDn9tH
+WjCVDa6wlw/Niq6BAmkE8d9ds2I8l0Xm1eHaMM3U3xY0OsmDYVP2p+BXZ7qWKa9c
+XjuT8gWTS0gZqerlALxTsIEy4/5iKhqdepjAefZxozS30kZhCMG7WXORV9pcdYFP
+rCoVPES85sAfwjjL9ZxmtoqH5845KoTlZWqbI/NJ/KCNa1VGXcc7NuNnCUo8sWqe
+kTwFSOnF+kaXtDFjM5/7/eQWKBelWWXysMX2+pUCQdIcUa5LW3M+16AjF906+DGZ
+pptUebilOd7CEXFKwgO2dZXLkTXj5hyKHYyTt066jPIdyAfGZe9oF0ttzwSS74WY
+Y1Sx1PvAH8B5+jfGnYKhVZHbX0nzdBvwG3FNlg2+GVrpTynTH1l1pVUV8YWrbWhh
+JE+xjLk0RKfC9jmhs3EenpfpYAEkIKZO3CGVXhZMi4kd7wUZud9vGjOcBlOF3YGG
+cVjYDRAymlY1VH3hvkToMZPdjJk8+1fT0bbWTXXjppV3tpC9aybz4H3BOvTXh8MN
+c7X4Pn1rDgjtPK2HfvuR6t9+LqWYTM15NeTnEtdkDdQGUmr3CYQI2h07bQYjtGDY
+XCfYZ4rRLYGcXiRKmm+NGGb/rsJcJe0KeVPZZmIFP5gfvmWvaQeY4lYw1YABdh9Y
+gTIqd+T4OGB5S9EIGrG6uXrlJkCZnIxOJjBPGkVsygn2QOdkIJ8tnycXB3ChTBfL
+FMA3i59W/pGf9apHpGF+iA==
diff --git a/packages/google-resumable-media/tests/system/requests/__init__.py b/packages/google-resumable-media/tests/system/requests/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests/system/requests/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests/system/requests/conftest.py b/packages/google-resumable-media/tests/system/requests/conftest.py
new file mode 100644
index 000000000000..54ae10a2b5ca
--- /dev/null
+++ b/packages/google-resumable-media/tests/system/requests/conftest.py
@@ -0,0 +1,58 @@
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""py.test fixtures to be shared across multiple system test modules."""
+
+import google.auth # type: ignore
+import google.auth.transport.requests as tr_requests # type: ignore
+import pytest # type: ignore
+
+from tests.system import utils
+
+
+def ensure_bucket(transport):
+ get_response = transport.get(utils.BUCKET_URL)
+ if get_response.status_code == 404:
+ credentials = transport.credentials
+ query_params = {"project": credentials.project_id}
+ payload = {"name": utils.BUCKET_NAME}
+ post_response = transport.post(
+ utils.BUCKET_POST_URL, params=query_params, json=payload
+ )
+
+ if not post_response.ok:
+ raise ValueError(
+ "{}: {}".format(post_response.status_code, post_response.reason)
+ )
+
+
+def cleanup_bucket(transport):
+ del_response = utils.retry_transient_errors(transport.delete)(utils.BUCKET_URL)
+
+ if not del_response.ok:
+ raise ValueError("{}: {}".format(del_response.status_code, del_response.reason))
+
+
+@pytest.fixture(scope="session")
+def authorized_transport():
+ credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,))
+ yield tr_requests.AuthorizedSession(credentials)
+
+
+@pytest.fixture(scope="session")
+def bucket(authorized_transport):
+ ensure_bucket(authorized_transport)
+
+ yield utils.BUCKET_NAME
+
+ cleanup_bucket(authorized_transport)
diff --git a/packages/google-resumable-media/tests/system/requests/test_download.py b/packages/google-resumable-media/tests/system/requests/test_download.py
new file mode 100644
index 000000000000..f0bd6c7e30e1
--- /dev/null
+++ b/packages/google-resumable-media/tests/system/requests/test_download.py
@@ -0,0 +1,634 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import copy
+import hashlib
+import http.client
+import io
+import os
+
+import google.auth # type: ignore
+import google.auth.transport.requests as tr_requests # type: ignore
+import pytest # type: ignore
+
+from google.resumable_media import common
+import google.resumable_media.requests as resumable_requests
+from google.resumable_media import _helpers
+from google.resumable_media.requests import _request_helpers
+import google.resumable_media.requests.download as download_mod
+from tests.system import utils
+
+
+CURR_DIR = os.path.dirname(os.path.realpath(__file__))
+DATA_DIR = os.path.join(CURR_DIR, "..", "..", "data")
+PLAIN_TEXT = "text/plain"
+IMAGE_JPEG = "image/jpeg"
+ENCRYPTED_ERR = b"The target object is encrypted by a customer-supplied encryption key."
+NO_BODY_ERR = "The content for this response was already consumed"
+NOT_FOUND_ERR = (
+ b"No such object: " + utils.BUCKET_NAME.encode("utf-8") + b"/does-not-exist.txt"
+)
+SIMPLE_DOWNLOADS = (resumable_requests.Download, resumable_requests.RawDownload)
+
+
+class CorruptingAuthorizedSession(tr_requests.AuthorizedSession):
+ """A Requests Session class with credentials, which corrupts responses.
+
+ This class is used for testing checksum validation.
+
+ Args:
+ credentials (google.auth.credentials.Credentials): The credentials to
+ add to the request.
+ refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
+ that credentials should be refreshed and the request should be
+ retried.
+ max_refresh_attempts (int): The maximum number of times to attempt to
+ refresh the credentials and retry the request.
+ kwargs: Additional arguments passed to the :class:`requests.Session`
+ constructor.
+ """
+
+ EMPTY_MD5 = base64.b64encode(hashlib.md5(b"").digest()).decode("utf-8")
+ crc32c = _helpers._get_crc32c_object()
+ crc32c.update(b"")
+ EMPTY_CRC32C = base64.b64encode(crc32c.digest()).decode("utf-8")
+
+ def request(self, method, url, data=None, headers=None, **kwargs):
+ """Implementation of Requests' request."""
+ response = tr_requests.AuthorizedSession.request(
+ self, method, url, data=data, headers=headers, **kwargs
+ )
+ response.headers[_helpers._HASH_HEADER] = "crc32c={},md5={}".format(
+ self.EMPTY_CRC32C, self.EMPTY_MD5
+ )
+ return response
+
+
+def get_path(filename):
+ return os.path.realpath(os.path.join(DATA_DIR, filename))
+
+
+ALL_FILES = (
+ {
+ "path": get_path("image1.jpg"),
+ "content_type": IMAGE_JPEG,
+ "md5": "1bsd83IYNug8hd+V1ING3Q==",
+ "crc32c": "YQGPxA==",
+ "slices": (
+ slice(1024, 16386, None), # obj[1024:16386]
+ slice(None, 8192, None), # obj[:8192]
+ slice(-256, None, None), # obj[-256:]
+ slice(262144, None, None), # obj[262144:]
+ ),
+ },
+ {
+ "path": get_path("image2.jpg"),
+ "content_type": IMAGE_JPEG,
+ "md5": "gdLXJltiYAMP9WZZFEQI1Q==",
+ "crc32c": "sxxEFQ==",
+ "slices": (
+ slice(1024, 16386, None), # obj[1024:16386]
+ slice(None, 8192, None), # obj[:8192]
+ slice(-256, None, None), # obj[-256:]
+ slice(262144, None, None), # obj[262144:]
+ ),
+ },
+ {
+ "path": get_path("file.txt"),
+ "content_type": PLAIN_TEXT,
+ "md5": "XHSHAr/SpIeZtZbjgQ4nGw==",
+ "crc32c": "MeMHoQ==",
+ "slices": (),
+ },
+ {
+ "path": get_path("gzipped.txt.gz"),
+ "uncompressed": get_path("gzipped.txt"),
+ "content_type": PLAIN_TEXT,
+ "md5": "KHRs/+ZSrc/FuuR4qz/PZQ==",
+ "crc32c": "/LIRNg==",
+ "slices": (),
+ "metadata": {"contentEncoding": "gzip"},
+ },
+ {
+ "path": get_path("brotli.txt.br"),
+ "uncompressed": get_path("brotli.txt"),
+ "content_type": PLAIN_TEXT,
+ "md5": "MffJw7pTSX/7CVWFFPgwQA==",
+ "crc32c": "GGK0OQ==",
+ "slices": (),
+ "metadata": {"contentEncoding": "br"},
+ },
+)
+
+
+def get_contents_for_upload(info):
+ with open(info["path"], "rb") as file_obj:
+ return file_obj.read()
+
+
+def get_contents(info):
+ full_path = info.get("uncompressed", info["path"])
+ with open(full_path, "rb") as file_obj:
+ return file_obj.read()
+
+
+def get_raw_contents(info):
+ full_path = info["path"]
+ with open(full_path, "rb") as file_obj:
+ return file_obj.read()
+
+
+def get_blob_name(info):
+ full_path = info.get("uncompressed", info["path"])
+ return os.path.basename(full_path)
+
+
+def delete_blob(transport, blob_name):
+ metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
+ response = transport.delete(metadata_url)
+ assert response.status_code == http.client.NO_CONTENT
+
+
+@pytest.fixture(scope="module")
+def secret_file(authorized_transport, bucket):
+ blob_name = "super-seekrit.txt"
+ data = b"Please do not tell anyone my encrypted seekrit."
+
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ headers = utils.get_encryption_headers()
+ upload = resumable_requests.SimpleUpload(upload_url, headers=headers)
+ response = upload.transmit(authorized_transport, data, PLAIN_TEXT)
+ assert response.status_code == http.client.OK
+
+ yield blob_name, data, headers
+
+ delete_blob(authorized_transport, blob_name)
+
+
+# Transport that returns corrupt data, so we can exercise checksum handling.
+@pytest.fixture(scope="module")
+def corrupting_transport():
+ credentials, _ = google.auth.default(scopes=(utils.GCS_RW_SCOPE,))
+ yield CorruptingAuthorizedSession(credentials)
+
+
+@pytest.fixture(scope="module")
+def simple_file(authorized_transport, bucket):
+ blob_name = "basic-file.txt"
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ upload = resumable_requests.SimpleUpload(upload_url)
+ data = b"Simple contents"
+ response = upload.transmit(authorized_transport, data, PLAIN_TEXT)
+ assert response.status_code == http.client.OK
+
+ yield blob_name, data
+
+ delete_blob(authorized_transport, blob_name)
+
+
+@pytest.fixture(scope="module")
+def add_files(authorized_transport, bucket):
+ blob_names = []
+ for info in ALL_FILES:
+ to_upload = get_contents_for_upload(info)
+ blob_name = get_blob_name(info)
+
+ blob_names.append(blob_name)
+ if "metadata" in info:
+ upload = resumable_requests.MultipartUpload(utils.MULTIPART_UPLOAD)
+ metadata = copy.deepcopy(info["metadata"])
+ metadata["name"] = blob_name
+ response = upload.transmit(
+ authorized_transport, to_upload, metadata, info["content_type"]
+ )
+ else:
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ upload = resumable_requests.SimpleUpload(upload_url)
+ response = upload.transmit(
+ authorized_transport, to_upload, info["content_type"]
+ )
+
+ assert response.status_code == http.client.OK
+
+ yield
+
+ # Clean-up the blobs we created.
+ for blob_name in blob_names:
+ delete_blob(authorized_transport, blob_name)
+
+
+def check_tombstoned(download, transport):
+ assert download.finished
+ if isinstance(download, SIMPLE_DOWNLOADS):
+ with pytest.raises(ValueError) as exc_info:
+ download.consume(transport)
+ assert exc_info.match("A download can only be used once.")
+ else:
+ with pytest.raises(ValueError) as exc_info:
+ download.consume_next_chunk(transport)
+ assert exc_info.match("Download has finished.")
+
+
+def check_error_response(exc_info, status_code, message):
+ error = exc_info.value
+ response = error.response
+ assert response.status_code == status_code
+ assert response.content.startswith(message)
+ assert len(error.args) == 5
+ assert error.args[1] == status_code
+ assert error.args[3] == http.client.OK
+ assert error.args[4] == http.client.PARTIAL_CONTENT
+
+
+class TestDownload(object):
+ @staticmethod
+ def _get_target_class():
+ return resumable_requests.Download
+
+ def _make_one(self, media_url, **kw):
+ return self._get_target_class()(media_url, **kw)
+
+ @staticmethod
+ def _get_contents(info):
+ return get_contents(info)
+
+ @staticmethod
+ def _read_response_content(response):
+ return response.content
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ def test_download_full(self, add_files, authorized_transport, checksum):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = self._make_one(media_url, checksum=checksum)
+ # Consume the resource.
+ response = download.consume(authorized_transport)
+ assert response.status_code == http.client.OK
+ assert self._read_response_content(response) == actual_contents
+ check_tombstoned(download, authorized_transport)
+
+ def test_download_to_stream(self, add_files, authorized_transport):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, stream=stream)
+ # Consume the resource.
+ response = download.consume(authorized_transport)
+ assert response.status_code == http.client.OK
+ with pytest.raises(RuntimeError) as exc_info:
+ getattr(response, "content")
+ assert exc_info.value.args == (NO_BODY_ERR,)
+ assert response._content is False
+ assert response._content_consumed is True
+ assert stream.getvalue() == actual_contents
+ check_tombstoned(download, authorized_transport)
+
+ def test_download_gzip_w_stored_content_headers(
+ self, add_files, authorized_transport
+ ):
+ # Retrieve the gzip compressed file
+ info = ALL_FILES[-2]
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, stream=stream)
+ # Consume the resource.
+ response = download.consume(authorized_transport)
+ assert response.status_code == http.client.OK
+ assert response.headers.get(_helpers._STORED_CONTENT_ENCODING_HEADER) == "gzip"
+ assert response.headers.get("X-Goog-Stored-Content-Length") is not None
+ assert stream.getvalue() == actual_contents
+ check_tombstoned(download, authorized_transport)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test_download_brotli_w_stored_content_headers(
+ self, add_files, authorized_transport, checksum
+ ):
+ # Retrieve the br compressed file
+ info = ALL_FILES[-1]
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, stream=stream, checksum=checksum)
+ # Consume the resource.
+ response = download.consume(authorized_transport)
+ assert response.status_code == http.client.OK
+ assert response.headers.get(_helpers._STORED_CONTENT_ENCODING_HEADER) == "br"
+ assert response.headers.get("X-Goog-Stored-Content-Length") is not None
+ assert stream.getvalue() == actual_contents
+ check_tombstoned(download, authorized_transport)
+
+ def test_extra_headers(self, authorized_transport, secret_file):
+ blob_name, data, headers = secret_file
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = self._make_one(media_url, headers=headers)
+ # Consume the resource.
+ response = download.consume(authorized_transport)
+ assert response.status_code == http.client.OK
+ assert response.content == data
+ check_tombstoned(download, authorized_transport)
+ # Attempt to consume the resource **without** the headers.
+ download_wo = self._make_one(media_url)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download_wo.consume(authorized_transport)
+
+ check_error_response(exc_info, http.client.BAD_REQUEST, ENCRYPTED_ERR)
+ check_tombstoned(download_wo, authorized_transport)
+
+ def test_non_existent_file(self, authorized_transport, bucket):
+ blob_name = "does-not-exist.txt"
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = self._make_one(media_url)
+
+ # Try to consume the resource and fail.
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download.consume(authorized_transport)
+ check_error_response(exc_info, http.client.NOT_FOUND, NOT_FOUND_ERR)
+ check_tombstoned(download, authorized_transport)
+
+ def test_bad_range(self, simple_file, authorized_transport):
+ blob_name, data = simple_file
+ # Make sure we have an invalid range.
+ start = 32
+ end = 63
+ assert len(data) < start < end
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = self._make_one(media_url, start=start, end=end)
+
+ # Try to consume the resource and fail.
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download.consume(authorized_transport)
+
+ check_error_response(
+ exc_info,
+ http.client.REQUESTED_RANGE_NOT_SATISFIABLE,
+ b"Request range not satisfiable",
+ )
+ check_tombstoned(download, authorized_transport)
+
+ def _download_slice(self, media_url, slice_):
+ assert slice_.step is None
+
+ end = None
+ if slice_.stop is not None:
+ end = slice_.stop - 1
+
+ return self._make_one(media_url, start=slice_.start, end=end)
+
+ def test_download_partial(self, add_files, authorized_transport):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ for slice_ in info["slices"]:
+ download = self._download_slice(media_url, slice_)
+ response = download.consume(authorized_transport)
+ assert response.status_code == http.client.PARTIAL_CONTENT
+ assert response.content == actual_contents[slice_]
+ with pytest.raises(ValueError):
+ download.consume(authorized_transport)
+
+
+class TestRawDownload(TestDownload):
+ @staticmethod
+ def _get_target_class():
+ return resumable_requests.RawDownload
+
+ @staticmethod
+ def _get_contents(info):
+ return get_raw_contents(info)
+
+ @staticmethod
+ def _read_response_content(response):
+ return b"".join(
+ response.raw.stream(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
+ )
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test_corrupt_download(self, add_files, corrupting_transport, checksum):
+ for info in ALL_FILES:
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, stream=stream, checksum=checksum)
+ # Consume the resource.
+ with pytest.raises(common.DataCorruption) as exc_info:
+ download.consume(corrupting_transport)
+
+ assert download.finished
+
+ if checksum == "md5":
+ EMPTY_HASH = CorruptingAuthorizedSession.EMPTY_MD5
+ else:
+ EMPTY_HASH = CorruptingAuthorizedSession.EMPTY_CRC32C
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ download.media_url,
+ EMPTY_HASH,
+ info[checksum],
+ checksum_type=checksum.upper(),
+ )
+ assert msg in exc_info.value.args[0]
+
+ def test_corrupt_download_no_check(self, add_files, corrupting_transport):
+ for info in ALL_FILES:
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, stream=stream, checksum=None)
+ # Consume the resource.
+ download.consume(corrupting_transport)
+
+ assert download.finished
+
+
+def get_chunk_size(min_chunks, total_bytes):
+ # Make sure the number of chunks **DOES NOT** evenly divide.
+ num_chunks = min_chunks
+ while total_bytes % num_chunks == 0:
+ num_chunks += 1
+
+ chunk_size = total_bytes // num_chunks
+ # Since we know an integer division has remainder, increment by 1.
+ chunk_size += 1
+ assert total_bytes < num_chunks * chunk_size
+
+ return num_chunks, chunk_size
+
+
+def consume_chunks(download, authorized_transport, total_bytes, actual_contents):
+ start_byte = download.start
+ end_byte = download.end
+ if end_byte is None:
+ end_byte = total_bytes - 1
+
+ num_responses = 0
+ while not download.finished:
+ response = download.consume_next_chunk(authorized_transport)
+ num_responses += 1
+
+ next_byte = min(start_byte + download.chunk_size, end_byte + 1)
+ assert download.bytes_downloaded == next_byte - download.start
+ assert download.total_bytes == total_bytes
+ assert response.status_code == http.client.PARTIAL_CONTENT
+ assert response.content == actual_contents[start_byte:next_byte]
+ start_byte = next_byte
+
+ return num_responses, response
+
+
+class TestChunkedDownload(object):
+ @staticmethod
+ def _get_target_class():
+ return resumable_requests.ChunkedDownload
+
+ def _make_one(self, media_url, chunk_size, stream, **kw):
+ return self._get_target_class()(media_url, chunk_size, stream, **kw)
+
+ @staticmethod
+ def _get_contents(info):
+ return get_contents(info)
+
+ def test_chunked_download_partial(self, add_files, authorized_transport):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ for slice_ in info["slices"]:
+ # Manually replace a missing start with 0.
+ start = 0 if slice_.start is None else slice_.start
+ # Chunked downloads don't support a negative index.
+ if start < 0:
+ continue
+
+ # First determine how much content is in the slice and
+ # use it to determine a chunking strategy.
+ total_bytes = len(actual_contents)
+ if slice_.stop is None:
+ end_byte = total_bytes - 1
+ end = None
+ else:
+ # Python slices DO NOT include the last index, though a byte
+ # range **is** inclusive of both endpoints.
+ end_byte = slice_.stop - 1
+ end = end_byte
+
+ num_chunks, chunk_size = get_chunk_size(7, end_byte - start + 1)
+ # Create the actual download object.
+ stream = io.BytesIO()
+ download = self._make_one(
+ media_url, chunk_size, stream, start=start, end=end
+ )
+ # Consume the resource in chunks.
+ num_responses, last_response = consume_chunks(
+ download, authorized_transport, total_bytes, actual_contents
+ )
+
+ # Make sure the combined chunks are the whole slice.
+ assert stream.getvalue() == actual_contents[slice_]
+ # Check that we have the right number of responses.
+ assert num_responses == num_chunks
+ # Make sure the last chunk isn't the same size.
+ assert len(last_response.content) < chunk_size
+ check_tombstoned(download, authorized_transport)
+
+ def test_chunked_with_extra_headers(self, authorized_transport, secret_file):
+ blob_name, data, headers = secret_file
+ num_chunks = 4
+ chunk_size = 12
+ assert (num_chunks - 1) * chunk_size < len(data) < num_chunks * chunk_size
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, chunk_size, stream, headers=headers)
+ # Consume the resource in chunks.
+ num_responses, last_response = consume_chunks(
+ download, authorized_transport, len(data), data
+ )
+ # Make sure the combined chunks are the whole object.
+ assert stream.getvalue() == data
+ # Check that we have the right number of responses.
+ assert num_responses == num_chunks
+ # Make sure the last chunk isn't the same size.
+ assert len(last_response.content) < chunk_size
+ check_tombstoned(download, authorized_transport)
+ # Attempt to consume the resource **without** the headers.
+ stream_wo = io.BytesIO()
+ download_wo = resumable_requests.ChunkedDownload(
+ media_url, chunk_size, stream_wo
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download_wo.consume_next_chunk(authorized_transport)
+
+ assert stream_wo.tell() == 0
+ check_error_response(exc_info, http.client.BAD_REQUEST, ENCRYPTED_ERR)
+ assert download_wo.invalid
+
+
+class TestRawChunkedDownload(TestChunkedDownload):
+ @staticmethod
+ def _get_target_class():
+ return resumable_requests.RawChunkedDownload
+
+ @staticmethod
+ def _get_contents(info):
+ return get_raw_contents(info)
+
+ def test_chunked_download_full(self, add_files, authorized_transport):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ total_bytes = len(actual_contents)
+ num_chunks, chunk_size = get_chunk_size(7, total_bytes)
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, chunk_size, stream)
+ # Consume the resource in chunks.
+ num_responses, last_response = consume_chunks(
+ download, authorized_transport, total_bytes, actual_contents
+ )
+ # Make sure the combined chunks are the whole object.
+ assert stream.getvalue() == actual_contents
+ # Check that we have the right number of responses.
+ assert num_responses == num_chunks
+ # Make sure the last chunk isn't the same size.
+ assert total_bytes % chunk_size != 0
+ assert len(last_response.content) < chunk_size
+ check_tombstoned(download, authorized_transport)
diff --git a/packages/google-resumable-media/tests/system/requests/test_upload.py b/packages/google-resumable-media/tests/system/requests/test_upload.py
new file mode 100644
index 000000000000..3f961bc4e754
--- /dev/null
+++ b/packages/google-resumable-media/tests/system/requests/test_upload.py
@@ -0,0 +1,776 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import hashlib
+import http.client
+import io
+import os
+import urllib.parse
+
+import pytest # type: ignore
+from unittest import mock
+
+from google.resumable_media import common
+from google import resumable_media
+import google.resumable_media.requests as resumable_requests
+from google.resumable_media import _helpers
+from tests.system import utils
+from google.resumable_media import _upload
+
+
+CURR_DIR = os.path.dirname(os.path.realpath(__file__))
+DATA_DIR = os.path.join(CURR_DIR, "..", "..", "data")
+ICO_FILE = os.path.realpath(os.path.join(DATA_DIR, "favicon.ico"))
+IMAGE_FILE = os.path.realpath(os.path.join(DATA_DIR, "image1.jpg"))
+ICO_CONTENT_TYPE = "image/x-icon"
+JPEG_CONTENT_TYPE = "image/jpeg"
+BYTES_CONTENT_TYPE = "application/octet-stream"
+BAD_CHUNK_SIZE_MSG = (
+ b"Invalid request. The number of bytes uploaded is required to be equal "
+ b"or greater than 262144, except for the final request (it's recommended "
+ b"to be the exact multiple of 262144). The received request contained "
+ b"1024 bytes, which does not meet this requirement."
+)
+
+
+@pytest.fixture
+def cleanup():
+ to_delete = []
+
+ def add_cleanup(blob_name, transport):
+ to_delete.append((blob_name, transport))
+
+ yield add_cleanup
+
+ for blob_name, transport in to_delete:
+ metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
+ response = utils.retry_transient_errors(transport.delete)(metadata_url)
+ assert response.status_code == http.client.NO_CONTENT
+
+
+@pytest.fixture
+def img_stream():
+ """Open-file as a fixture.
+
+ This is so that an entire test can execute in the context of
+ the context manager without worrying about closing the file.
+ """
+ with open(IMAGE_FILE, "rb") as file_obj:
+ yield file_obj
+
+
+def get_md5(data):
+ hash_obj = hashlib.md5(data)
+ return base64.b64encode(hash_obj.digest())
+
+
+def get_upload_id(upload_url):
+ parse_result = urllib.parse.urlparse(upload_url)
+ parsed_query = urllib.parse.parse_qs(parse_result.query)
+ # NOTE: We are unpacking here, so asserting exactly one match.
+ (upload_id,) = parsed_query["upload_id"]
+ return upload_id
+
+
+def get_num_chunks(total_bytes, chunk_size):
+ expected_chunks, remainder = divmod(total_bytes, chunk_size)
+ if remainder > 0:
+ expected_chunks += 1
+ return expected_chunks
+
+
+def check_response(
+ response,
+ blob_name,
+ actual_contents=None,
+ total_bytes=None,
+ metadata=None,
+ content_type=ICO_CONTENT_TYPE,
+):
+ assert response.status_code == http.client.OK
+ json_response = response.json()
+ assert json_response["bucket"] == utils.BUCKET_NAME
+ assert json_response["contentType"] == content_type
+ if actual_contents is not None:
+ md5_hash = json_response["md5Hash"].encode("ascii")
+ assert md5_hash == get_md5(actual_contents)
+ total_bytes = len(actual_contents)
+ assert json_response["metageneration"] == "1"
+ assert json_response["name"] == blob_name
+ assert json_response["size"] == "{:d}".format(total_bytes)
+ assert json_response["storageClass"] == "STANDARD"
+ if metadata is None:
+ assert "metadata" not in json_response
+ else:
+ assert json_response["metadata"] == metadata
+
+
+def check_content(blob_name, expected_content, transport, headers=None):
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = resumable_requests.Download(media_url, headers=headers)
+ response = download.consume(transport)
+ assert response.status_code == http.client.OK
+ assert response.content == expected_content
+
+
+def check_tombstoned(upload, transport, *args):
+ assert upload.finished
+ basic_types = (resumable_requests.SimpleUpload, resumable_requests.MultipartUpload)
+ if isinstance(upload, basic_types):
+ with pytest.raises(ValueError):
+ upload.transmit(transport, *args)
+ else:
+ with pytest.raises(ValueError):
+ upload.transmit_next_chunk(transport, *args)
+
+
+def check_does_not_exist(transport, blob_name):
+ metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
+ # Make sure we are creating a **new** object.
+ response = transport.get(metadata_url)
+ assert response.status_code == http.client.NOT_FOUND
+
+
+def check_initiate(response, upload, stream, transport, metadata):
+ assert response.status_code == http.client.OK
+ assert response.content == b""
+ upload_id = get_upload_id(upload.resumable_url)
+ assert response.headers["x-guploader-uploadid"] == upload_id
+ assert stream.tell() == 0
+ # Make sure the upload cannot be re-initiated.
+ with pytest.raises(ValueError) as exc_info:
+ upload.initiate(transport, stream, metadata, JPEG_CONTENT_TYPE)
+
+ exc_info.match("This upload has already been initiated.")
+
+
+def check_bad_chunk(upload, transport):
+ with pytest.raises(resumable_media.InvalidResponse) as exc_info:
+ upload.transmit_next_chunk(transport)
+ error = exc_info.value
+ response = error.response
+ assert response.status_code == http.client.BAD_REQUEST
+ assert response.content == BAD_CHUNK_SIZE_MSG
+
+
+def transmit_chunks(
+ upload, transport, blob_name, metadata, num_chunks=0, content_type=JPEG_CONTENT_TYPE
+):
+ while not upload.finished:
+ num_chunks += 1
+ response = upload.transmit_next_chunk(transport)
+ if upload.finished:
+ assert upload.bytes_uploaded == upload.total_bytes
+ check_response(
+ response,
+ blob_name,
+ total_bytes=upload.total_bytes,
+ metadata=metadata,
+ content_type=content_type,
+ )
+ else:
+ assert upload.bytes_uploaded == num_chunks * upload.chunk_size
+ assert response.status_code == http.client.PERMANENT_REDIRECT
+
+ return num_chunks
+
+
+def test_simple_upload(authorized_transport, bucket, cleanup):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ upload = resumable_requests.SimpleUpload(upload_url)
+ # Transmit the resource.
+ response = upload.transmit(authorized_transport, actual_contents, ICO_CONTENT_TYPE)
+ check_response(response, blob_name, actual_contents=actual_contents)
+ # Download the content to make sure it's "working as expected".
+ check_content(blob_name, actual_contents, authorized_transport)
+ # Make sure the upload is tombstoned.
+ check_tombstoned(upload, authorized_transport, actual_contents, ICO_CONTENT_TYPE)
+
+
+def test_simple_upload_with_headers(authorized_transport, bucket, cleanup):
+ blob_name = "some-stuff.bin"
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ headers = utils.get_encryption_headers()
+ upload = resumable_requests.SimpleUpload(upload_url, headers=headers)
+ # Transmit the resource.
+ data = b"Binary contents\x00\x01\x02."
+ response = upload.transmit(authorized_transport, data, BYTES_CONTENT_TYPE)
+ check_response(
+ response, blob_name, actual_contents=data, content_type=BYTES_CONTENT_TYPE
+ )
+ # Download the content to make sure it's "working as expected".
+ check_content(blob_name, data, authorized_transport, headers=headers)
+ # Make sure the upload is tombstoned.
+ check_tombstoned(upload, authorized_transport, data, BYTES_CONTENT_TYPE)
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+def test_multipart_upload(authorized_transport, bucket, cleanup, checksum):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.MULTIPART_UPLOAD
+ upload = resumable_requests.MultipartUpload(upload_url, checksum=checksum)
+ # Transmit the resource.
+ metadata = {"name": blob_name, "metadata": {"color": "yellow"}}
+ response = upload.transmit(
+ authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE
+ )
+ check_response(
+ response,
+ blob_name,
+ actual_contents=actual_contents,
+ metadata=metadata["metadata"],
+ )
+ # Download the content to make sure it's "working as expected".
+ check_content(blob_name, actual_contents, authorized_transport)
+ # Make sure the upload is tombstoned.
+ check_tombstoned(
+ upload, authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE
+ )
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+def test_multipart_upload_with_bad_checksum(authorized_transport, checksum, bucket):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.MULTIPART_UPLOAD
+ upload = resumable_requests.MultipartUpload(upload_url, checksum=checksum)
+ # Transmit the resource.
+ metadata = {"name": blob_name, "metadata": {"color": "yellow"}}
+ fake_checksum_object = _helpers._get_checksum_object(checksum)
+ fake_checksum_object.update(b"bad data")
+ fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
+ fake_checksum_object.digest()
+ )
+ with mock.patch.object(
+ _helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest
+ ):
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ response = upload.transmit(
+ authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE
+ )
+ response = exc_info.value.response
+ message = response.json()["error"]["message"]
+ # Attempt to verify that this is a checksum mismatch error.
+ assert checksum.upper() in message
+ assert fake_prepared_checksum_digest in message
+
+ # Make sure the upload is tombstoned.
+ check_tombstoned(
+ upload, authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE
+ )
+
+
+def test_multipart_upload_with_headers(authorized_transport, bucket, cleanup):
+ blob_name = "some-multipart-stuff.bin"
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.MULTIPART_UPLOAD
+ headers = utils.get_encryption_headers()
+ upload = resumable_requests.MultipartUpload(upload_url, headers=headers)
+ # Transmit the resource.
+ metadata = {"name": blob_name}
+ data = b"Other binary contents\x03\x04\x05."
+ response = upload.transmit(authorized_transport, data, metadata, BYTES_CONTENT_TYPE)
+ check_response(
+ response, blob_name, actual_contents=data, content_type=BYTES_CONTENT_TYPE
+ )
+ # Download the content to make sure it's "working as expected".
+ check_content(blob_name, data, authorized_transport, headers=headers)
+ # Make sure the upload is tombstoned.
+ check_tombstoned(upload, authorized_transport, data, metadata, BYTES_CONTENT_TYPE)
+
+
+def _resumable_upload_helper(
+ authorized_transport, stream, cleanup, headers=None, checksum=None
+):
+ blob_name = os.path.basename(stream.name)
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+ # Create the actual upload object.
+ chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, chunk_size, headers=headers, checksum=checksum
+ )
+ # Initiate the upload.
+ metadata = {"name": blob_name, "metadata": {"direction": "north"}}
+ response = upload.initiate(
+ authorized_transport, stream, metadata, JPEG_CONTENT_TYPE
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Actually upload the file in chunks.
+ num_chunks = transmit_chunks(
+ upload, authorized_transport, blob_name, metadata["metadata"]
+ )
+ assert num_chunks == get_num_chunks(upload.total_bytes, chunk_size)
+ # Download the content to make sure it's "working as expected".
+ stream.seek(0)
+ actual_contents = stream.read()
+ check_content(blob_name, actual_contents, authorized_transport, headers=headers)
+ # Make sure the upload is tombstoned.
+ check_tombstoned(upload, authorized_transport)
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+def test_resumable_upload(authorized_transport, img_stream, bucket, cleanup, checksum):
+ _resumable_upload_helper(
+ authorized_transport, img_stream, cleanup, checksum=checksum
+ )
+
+
+def test_resumable_upload_with_headers(
+ authorized_transport, img_stream, bucket, cleanup
+):
+ headers = utils.get_encryption_headers()
+ _resumable_upload_helper(authorized_transport, img_stream, cleanup, headers=headers)
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+def test_resumable_upload_with_bad_checksum(
+ authorized_transport, img_stream, bucket, cleanup, checksum
+):
+ fake_checksum_object = _helpers._get_checksum_object(checksum)
+ fake_checksum_object.update(b"bad data")
+ fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
+ fake_checksum_object.digest()
+ )
+ with mock.patch.object(
+ _helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest
+ ):
+ with pytest.raises(common.DataCorruption) as exc_info:
+ _resumable_upload_helper(
+ authorized_transport, img_stream, cleanup, checksum=checksum
+ )
+ expected_checksums = {"md5": "1bsd83IYNug8hd+V1ING3Q==", "crc32c": "YQGPxA=="}
+ expected_message = _upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
+ checksum.upper(), fake_prepared_checksum_digest, expected_checksums[checksum]
+ )
+ assert exc_info.value.args[0] == expected_message
+
+
+def test_resumable_upload_bad_chunk_size(authorized_transport, img_stream):
+ blob_name = os.path.basename(img_stream.name)
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, resumable_media.UPLOAD_CHUNK_SIZE
+ )
+ # Modify the ``upload`` **after** construction so we can
+ # use a bad chunk size.
+ upload._chunk_size = 1024
+ assert upload._chunk_size < resumable_media.UPLOAD_CHUNK_SIZE
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ response = upload.initiate(
+ authorized_transport, img_stream, metadata, JPEG_CONTENT_TYPE
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ check_initiate(response, upload, img_stream, authorized_transport, metadata)
+ # Make the first request and verify that it fails.
+ check_bad_chunk(upload, authorized_transport)
+ # Reset the chunk size (and the stream) and verify the "resumable"
+ # URL is unusable.
+ upload._chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
+ img_stream.seek(0)
+ upload._invalid = False
+ check_bad_chunk(upload, authorized_transport)
+
+
+def sabotage_and_recover(upload, stream, transport, chunk_size):
+ assert upload.bytes_uploaded == chunk_size
+ assert stream.tell() == chunk_size
+ # "Fake" that the instance is in an invalid state.
+ upload._invalid = True
+ stream.seek(0) # Seek to the wrong place.
+ upload._bytes_uploaded = 0 # Make ``bytes_uploaded`` wrong as well.
+ # Recover the (artifically) invalid upload.
+ response = upload.recover(transport)
+ assert response.status_code == http.client.PERMANENT_REDIRECT
+ assert not upload.invalid
+ assert upload.bytes_uploaded == chunk_size
+ assert stream.tell() == chunk_size
+
+
+def _resumable_upload_recover_helper(
+ authorized_transport, cleanup, headers=None, checksum=None
+):
+ blob_name = "some-bytes.bin"
+ chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
+ data = b"123" * chunk_size # 3 chunks worth.
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, chunk_size, headers=headers, checksum=checksum
+ )
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ stream = io.BytesIO(data)
+ response = upload.initiate(
+ authorized_transport, stream, metadata, BYTES_CONTENT_TYPE
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Make the first request.
+ response = upload.transmit_next_chunk(authorized_transport)
+ assert response.status_code == http.client.PERMANENT_REDIRECT
+ # Call upload.recover().
+ sabotage_and_recover(upload, stream, authorized_transport, chunk_size)
+ # Now stream what remains.
+ num_chunks = transmit_chunks(
+ upload,
+ authorized_transport,
+ blob_name,
+ None,
+ num_chunks=1,
+ content_type=BYTES_CONTENT_TYPE,
+ )
+ assert num_chunks == 3
+ # Download the content to make sure it's "working as expected".
+ actual_contents = stream.getvalue()
+ check_content(blob_name, actual_contents, authorized_transport, headers=headers)
+ # Make sure the upload is tombstoned.
+ check_tombstoned(upload, authorized_transport)
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+def test_resumable_upload_recover(authorized_transport, bucket, cleanup, checksum):
+ _resumable_upload_recover_helper(authorized_transport, cleanup, checksum=checksum)
+
+
+def test_resumable_upload_recover_with_headers(authorized_transport, bucket, cleanup):
+ headers = utils.get_encryption_headers()
+ _resumable_upload_recover_helper(authorized_transport, cleanup, headers=headers)
+
+
+class TestResumableUploadUnknownSize(object):
+ @staticmethod
+ def _check_range_sent(response, start, end, total):
+ headers_sent = response.request.headers
+ if start is None and end is None:
+ expected_content_range = "bytes */{:d}".format(total)
+ else:
+ # Allow total to be an int or a string "*"
+ expected_content_range = "bytes {:d}-{:d}/{}".format(start, end, total)
+
+ assert headers_sent["content-range"] == expected_content_range
+
+ @staticmethod
+ def _check_range_received(response, size):
+ assert response.headers["range"] == "bytes=0-{:d}".format(size - 1)
+
+ def _check_partial(self, upload, response, chunk_size, num_chunks):
+ start_byte = (num_chunks - 1) * chunk_size
+ end_byte = num_chunks * chunk_size - 1
+
+ assert not upload.finished
+ assert upload.bytes_uploaded == end_byte + 1
+ assert response.status_code == http.client.PERMANENT_REDIRECT
+ assert response.content == b""
+
+ self._check_range_sent(response, start_byte, end_byte, "*")
+ self._check_range_received(response, end_byte + 1)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ def test_smaller_than_chunk_size(
+ self, authorized_transport, bucket, cleanup, checksum
+ ):
+ blob_name = os.path.basename(ICO_FILE)
+ chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+ # Make sure the blob is smaller than the chunk size.
+ total_bytes = os.path.getsize(ICO_FILE)
+ assert total_bytes < chunk_size
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, chunk_size, checksum=checksum
+ )
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ with open(ICO_FILE, "rb") as stream:
+ response = upload.initiate(
+ authorized_transport,
+ stream,
+ metadata,
+ ICO_CONTENT_TYPE,
+ stream_final=False,
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Make sure total bytes was never set.
+ assert upload.total_bytes is None
+ # Make the **ONLY** request.
+ response = upload.transmit_next_chunk(authorized_transport)
+ self._check_range_sent(response, 0, total_bytes - 1, total_bytes)
+ check_response(response, blob_name, total_bytes=total_bytes)
+ # Download the content to make sure it's "working as expected".
+ stream.seek(0)
+ actual_contents = stream.read()
+ check_content(blob_name, actual_contents, authorized_transport)
+ # Make sure the upload is tombstoned.
+ check_tombstoned(upload, authorized_transport)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ def test_finish_at_chunk(self, authorized_transport, bucket, cleanup, checksum):
+ blob_name = "some-clean-stuff.bin"
+ chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+ # Make sure the blob size is an exact multiple of the chunk size.
+ data = b"ab" * chunk_size
+ total_bytes = len(data)
+ stream = io.BytesIO(data)
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, chunk_size, checksum=checksum
+ )
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ response = upload.initiate(
+ authorized_transport,
+ stream,
+ metadata,
+ BYTES_CONTENT_TYPE,
+ stream_final=False,
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Make sure total bytes was never set.
+ assert upload.total_bytes is None
+ # Make three requests.
+ response0 = upload.transmit_next_chunk(authorized_transport)
+ self._check_partial(upload, response0, chunk_size, 1)
+
+ response1 = upload.transmit_next_chunk(authorized_transport)
+ self._check_partial(upload, response1, chunk_size, 2)
+
+ response2 = upload.transmit_next_chunk(authorized_transport)
+ assert upload.finished
+ # Verify the "clean-up" request.
+ assert upload.bytes_uploaded == 2 * chunk_size
+ check_response(
+ response2,
+ blob_name,
+ actual_contents=data,
+ total_bytes=total_bytes,
+ content_type=BYTES_CONTENT_TYPE,
+ )
+ self._check_range_sent(response2, None, None, 2 * chunk_size)
+
+ @staticmethod
+ def _add_bytes(stream, data):
+ curr_pos = stream.tell()
+ stream.write(data)
+ # Go back to where we were before the write.
+ stream.seek(curr_pos)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ def test_interleave_writes(self, authorized_transport, bucket, cleanup, checksum):
+ blob_name = "some-moar-stuff.bin"
+ chunk_size = resumable_media.UPLOAD_CHUNK_SIZE
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+ # Start out the blob as a single chunk (but we will add to it).
+ stream = io.BytesIO(b"Z" * chunk_size)
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, chunk_size, checksum=checksum
+ )
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ response = upload.initiate(
+ authorized_transport,
+ stream,
+ metadata,
+ BYTES_CONTENT_TYPE,
+ stream_final=False,
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Make sure total bytes was never set.
+ assert upload.total_bytes is None
+ # Make three requests.
+ response0 = upload.transmit_next_chunk(authorized_transport)
+ self._check_partial(upload, response0, chunk_size, 1)
+ # Add another chunk before sending.
+ self._add_bytes(stream, b"K" * chunk_size)
+ response1 = upload.transmit_next_chunk(authorized_transport)
+ self._check_partial(upload, response1, chunk_size, 2)
+ # Add more bytes, but make sure less than a full chunk.
+ last_chunk = 155
+ self._add_bytes(stream, b"r" * last_chunk)
+ response2 = upload.transmit_next_chunk(authorized_transport)
+ assert upload.finished
+ # Verify the "clean-up" request.
+ total_bytes = 2 * chunk_size + last_chunk
+ assert upload.bytes_uploaded == total_bytes
+ check_response(
+ response2,
+ blob_name,
+ actual_contents=stream.getvalue(),
+ total_bytes=total_bytes,
+ content_type=BYTES_CONTENT_TYPE,
+ )
+ self._check_range_sent(response2, 2 * chunk_size, total_bytes - 1, total_bytes)
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+def test_XMLMPU(authorized_transport, bucket, cleanup, checksum):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ # Make sure to clean up the uploaded blob when we are done.
+ cleanup(blob_name, authorized_transport)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.XML_UPLOAD_URL_TEMPLATE.format(bucket=bucket, blob=blob_name)
+ container = resumable_requests.XMLMPUContainer(upload_url, blob_name)
+ # Initiate
+ container.initiate(authorized_transport, ICO_CONTENT_TYPE)
+ assert container.upload_id
+
+ part = resumable_requests.XMLMPUPart(
+ upload_url,
+ container.upload_id,
+ ICO_FILE,
+ 0,
+ len(actual_contents),
+ 1,
+ checksum=checksum,
+ )
+ part.upload(authorized_transport)
+ assert part.etag
+
+ container.register_part(1, part.etag)
+ container.finalize(authorized_transport)
+ assert container.finished
+
+ # Download the content to make sure it's "working as expected".
+ check_content(blob_name, actual_contents, authorized_transport)
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+def test_XMLMPU_with_bad_checksum(authorized_transport, bucket, checksum):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ # No need to clean up, since the upload will not be finalized successfully.
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.XML_UPLOAD_URL_TEMPLATE.format(bucket=bucket, blob=blob_name)
+ container = resumable_requests.XMLMPUContainer(upload_url, blob_name)
+ # Initiate
+ container.initiate(authorized_transport, ICO_CONTENT_TYPE)
+ assert container.upload_id
+
+ try:
+ part = resumable_requests.XMLMPUPart(
+ upload_url,
+ container.upload_id,
+ ICO_FILE,
+ 0,
+ len(actual_contents),
+ 1,
+ checksum=checksum,
+ )
+
+ fake_checksum_object = _helpers._get_checksum_object(checksum)
+ fake_checksum_object.update(b"bad data")
+ fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
+ fake_checksum_object.digest()
+ )
+ with mock.patch.object(
+ _helpers,
+ "prepare_checksum_digest",
+ return_value=fake_prepared_checksum_digest,
+ ):
+ with pytest.raises(common.DataCorruption):
+ part.upload(authorized_transport)
+ finally:
+ utils.retry_transient_errors(authorized_transport.delete)(
+ upload_url + "?uploadId=" + str(container.upload_id)
+ )
+
+
+def test_XMLMPU_cancel(authorized_transport, bucket):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.XML_UPLOAD_URL_TEMPLATE.format(bucket=bucket, blob=blob_name)
+ container = resumable_requests.XMLMPUContainer(upload_url, blob_name)
+ # Initiate
+ container.initiate(authorized_transport, ICO_CONTENT_TYPE)
+ assert container.upload_id
+
+ part = resumable_requests.XMLMPUPart(
+ upload_url,
+ container.upload_id,
+ ICO_FILE,
+ 0,
+ len(actual_contents),
+ 1,
+ )
+ part.upload(authorized_transport)
+ assert part.etag
+
+ container.register_part(1, part.etag)
+ container.cancel(authorized_transport)
+
+ # Validate the cancel worked by expecting a 404 on finalize.
+ with pytest.raises(resumable_media.InvalidResponse):
+ container.finalize(authorized_transport)
diff --git a/packages/google-resumable-media/tests/system/utils.py b/packages/google-resumable-media/tests/system/utils.py
new file mode 100644
index 000000000000..7b679095dac6
--- /dev/null
+++ b/packages/google-resumable-media/tests/system/utils.py
@@ -0,0 +1,88 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import hashlib
+import time
+
+from test_utils.retry import RetryResult # type: ignore
+
+
+BUCKET_NAME = "grpm-systest-{}".format(int(1000 * time.time()))
+BUCKET_POST_URL = "https://www.googleapis.com/storage/v1/b/"
+BUCKET_URL = "https://www.googleapis.com/storage/v1/b/{}".format(BUCKET_NAME)
+
+_DOWNLOAD_BASE = "https://www.googleapis.com/download/storage/v1/b/{}".format(
+ BUCKET_NAME
+)
+DOWNLOAD_URL_TEMPLATE = _DOWNLOAD_BASE + "/o/{blob_name}?alt=media"
+
+_UPLOAD_BASE = (
+ "https://www.googleapis.com/upload/storage/v1/b/{}".format(BUCKET_NAME)
+ + "/o?uploadType="
+)
+SIMPLE_UPLOAD_TEMPLATE = _UPLOAD_BASE + "media&name={blob_name}"
+MULTIPART_UPLOAD = _UPLOAD_BASE + "multipart"
+RESUMABLE_UPLOAD = _UPLOAD_BASE + "resumable"
+
+METADATA_URL_TEMPLATE = BUCKET_URL + "/o/{blob_name}"
+
+XML_UPLOAD_URL_TEMPLATE = "https://{bucket}.storage.googleapis.com/{blob}"
+
+
+GCS_RW_SCOPE = "https://www.googleapis.com/auth/devstorage.read_write"
+# Generated using random.choice() with all 256 byte choices.
+ENCRYPTION_KEY = (
+ b"R\xb8\x1b\x94T\xea_\xa8\x93\xae\xd1\xf6\xfca\x15\x0ekA"
+ b"\x08 Y\x13\xe2\n\x02i\xadc\xe2\xd99x"
+)
+
+
+_RETRYABLE_CODES = [
+ 409, # Conflict
+ 429, # TooManyRequests
+ 503, # ServiceUnavailable
+]
+
+
+def _not_retryable(response):
+ return response.status_code not in _RETRYABLE_CODES
+
+
+retry_transient_errors = RetryResult(_not_retryable)
+
+
+def get_encryption_headers(key=ENCRYPTION_KEY):
+ """Builds customer-supplied encryption key headers
+
+ See `Managing Data Encryption`_ for more details.
+
+ Args:
+ key (bytes): 32 byte key to build request key and hash.
+
+ Returns:
+ Dict[str, str]: The algorithm, key and key-SHA256 headers.
+
+ .. _Managing Data Encryption:
+ https://cloud.google.com/storage/docs/encryption
+ """
+ key_hash = hashlib.sha256(key).digest()
+ key_hash_b64 = base64.b64encode(key_hash)
+ key_b64 = base64.b64encode(key)
+
+ return {
+ "x-goog-encryption-algorithm": "AES256",
+ "x-goog-encryption-key": key_b64.decode("utf-8"),
+ "x-goog-encryption-key-sha256": key_hash_b64.decode("utf-8"),
+ }
diff --git a/packages/google-resumable-media/tests/unit/__init__.py b/packages/google-resumable-media/tests/unit/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests/unit/requests/__init__.py b/packages/google-resumable-media/tests/unit/requests/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/requests/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests/unit/requests/test__helpers.py b/packages/google-resumable-media/tests/unit/requests/test__helpers.py
new file mode 100644
index 000000000000..de85991ac9e4
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/requests/test__helpers.py
@@ -0,0 +1,406 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+
+from unittest import mock
+import pytest # type: ignore
+
+import requests.exceptions
+import urllib3.exceptions # type: ignore
+
+from google.resumable_media import common
+from google.resumable_media.requests import _request_helpers
+
+EXPECTED_TIMEOUT = (61, 60)
+
+
+class TestRequestsMixin(object):
+ def test__get_status_code(self):
+ status_code = int(http.client.OK)
+ response = _make_response(status_code)
+ assert status_code == _request_helpers.RequestsMixin._get_status_code(response)
+
+ def test__get_headers(self):
+ headers = {"fruit": "apple"}
+ response = mock.Mock(headers=headers, spec=["headers"])
+ assert headers == _request_helpers.RequestsMixin._get_headers(response)
+
+ def test__get_body(self):
+ body = b"This is the payload."
+ response = mock.Mock(content=body, spec=["content"])
+ assert body == _request_helpers.RequestsMixin._get_body(response)
+
+
+class TestRawRequestsMixin(object):
+ def test__get_body_wo_content_consumed(self):
+ body = b"This is the payload."
+ raw = mock.Mock(spec=["stream"])
+ raw.stream.return_value = iter([body])
+ response = mock.Mock(raw=raw, _content=False, spec=["raw", "_content"])
+ assert body == _request_helpers.RawRequestsMixin._get_body(response)
+ raw.stream.assert_called_once_with(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
+ )
+
+ def test__get_body_w_content_consumed(self):
+ body = b"This is the payload."
+ response = mock.Mock(_content=body, spec=["_content"])
+ assert body == _request_helpers.RawRequestsMixin._get_body(response)
+
+
+def _make_response(status_code):
+ return mock.Mock(status_code=status_code, spec=["status_code"])
+
+
+def _get_status_code(response):
+ return response.status_code
+
+
+class Test_wait_and_retry(object):
+ def test_success_no_retry(self):
+ truthy = http.client.OK
+ assert truthy not in common.RETRYABLE
+ response = _make_response(truthy)
+
+ func = mock.Mock(return_value=response, spec=[])
+ retry_strategy = common.RetryStrategy()
+ ret_val = _request_helpers.wait_and_retry(
+ func, _get_status_code, retry_strategy
+ )
+
+ assert ret_val is response
+ func.assert_called_once_with()
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_success_with_retry(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [125, 625, 375]
+
+ status_codes = (
+ http.client.INTERNAL_SERVER_ERROR,
+ http.client.BAD_GATEWAY,
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.NOT_FOUND,
+ )
+ responses = [_make_response(status_code) for status_code in status_codes]
+
+ def raise_response():
+ raise common.InvalidResponse(responses.pop(0))
+
+ func = mock.Mock(side_effect=raise_response)
+
+ retry_strategy = common.RetryStrategy()
+ try:
+ _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+ except common.InvalidResponse as e:
+ ret_val = e.response
+
+ assert ret_val.status_code == status_codes[-1]
+ assert status_codes[-1] not in common.RETRYABLE
+
+ assert func.call_count == 4
+ assert func.mock_calls == [mock.call()] * 4
+
+ assert randint_mock.call_count == 3
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3
+
+ assert sleep_mock.call_count == 3
+ sleep_mock.assert_any_call(1.125)
+ sleep_mock.assert_any_call(2.625)
+ sleep_mock.assert_any_call(4.375)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_success_with_retry_custom_delay(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [125, 625, 375]
+
+ status_codes = (
+ http.client.INTERNAL_SERVER_ERROR,
+ http.client.BAD_GATEWAY,
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.NOT_FOUND,
+ )
+ responses = [_make_response(status_code) for status_code in status_codes]
+
+ def raise_response():
+ raise common.InvalidResponse(responses.pop(0))
+
+ func = mock.Mock(side_effect=raise_response)
+
+ retry_strategy = common.RetryStrategy(initial_delay=3.0, multiplier=4)
+ try:
+ _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+ except common.InvalidResponse as e:
+ ret_val = e.response
+
+ assert ret_val.status_code == status_codes[-1]
+ assert status_codes[-1] not in common.RETRYABLE
+
+ assert func.call_count == 4
+ assert func.mock_calls == [mock.call()] * 4
+
+ assert randint_mock.call_count == 3
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3
+
+ assert sleep_mock.call_count == 3
+ sleep_mock.assert_any_call(3.125) # initial delay 3 + jitter 0.125
+ sleep_mock.assert_any_call(
+ 12.625
+ ) # previous delay 3 * multiplier 4 + jitter 0.625
+ sleep_mock.assert_any_call(
+ 48.375
+ ) # previous delay 12 * multiplier 4 + jitter 0.375
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_retry_success_http_standard_lib_connection_errors(
+ self, randint_mock, sleep_mock
+ ):
+ randint_mock.side_effect = [125, 625, 500, 875, 375]
+
+ status_code = int(http.client.OK)
+ response = _make_response(status_code)
+ responses = [
+ http.client.BadStatusLine(""),
+ http.client.IncompleteRead(""),
+ http.client.ResponseNotReady,
+ ConnectionError,
+ response,
+ ]
+ func = mock.Mock(side_effect=responses, spec=[])
+
+ retry_strategy = common.RetryStrategy()
+ ret_val = _request_helpers.wait_and_retry(
+ func, _get_status_code, retry_strategy
+ )
+
+ assert ret_val == responses[-1]
+ assert func.call_count == 5
+ assert func.mock_calls == [mock.call()] * 5
+ assert randint_mock.call_count == 4
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 4
+ assert sleep_mock.call_count == 4
+ sleep_mock.assert_any_call(1.125)
+ sleep_mock.assert_any_call(2.625)
+ sleep_mock.assert_any_call(4.500)
+ sleep_mock.assert_any_call(8.875)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_retry_success_requests_lib_connection_errors(
+ self, randint_mock, sleep_mock
+ ):
+ randint_mock.side_effect = [125, 625, 500, 875]
+
+ status_code = int(http.client.OK)
+ response = _make_response(status_code)
+ responses = [
+ requests.exceptions.ConnectionError,
+ requests.exceptions.ChunkedEncodingError,
+ requests.exceptions.Timeout,
+ response,
+ ]
+ func = mock.Mock(side_effect=responses, spec=[])
+
+ retry_strategy = common.RetryStrategy()
+ ret_val = _request_helpers.wait_and_retry(
+ func, _get_status_code, retry_strategy
+ )
+
+ assert ret_val == responses[-1]
+ assert func.call_count == 4
+ assert func.mock_calls == [mock.call()] * 4
+ assert randint_mock.call_count == 3
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3
+ assert sleep_mock.call_count == 3
+ sleep_mock.assert_any_call(1.125)
+ sleep_mock.assert_any_call(2.625)
+ sleep_mock.assert_any_call(4.500)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_retry_success_urllib3_connection_errors(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [125, 625, 500, 875, 375]
+
+ status_code = int(http.client.OK)
+ response = _make_response(status_code)
+ responses = [
+ urllib3.exceptions.PoolError(None, ""),
+ urllib3.exceptions.ProtocolError,
+ urllib3.exceptions.SSLError,
+ urllib3.exceptions.TimeoutError,
+ response,
+ ]
+ func = mock.Mock(side_effect=responses, spec=[])
+
+ retry_strategy = common.RetryStrategy()
+ ret_val = _request_helpers.wait_and_retry(
+ func, _get_status_code, retry_strategy
+ )
+
+ assert ret_val == responses[-1]
+ assert func.call_count == 5
+ assert func.mock_calls == [mock.call()] * 5
+ assert randint_mock.call_count == 4
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 4
+ assert sleep_mock.call_count == 4
+ sleep_mock.assert_any_call(1.125)
+ sleep_mock.assert_any_call(2.625)
+ sleep_mock.assert_any_call(4.500)
+ sleep_mock.assert_any_call(8.875)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_retry_exceeds_max_cumulative(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125]
+
+ status_codes = (
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.GATEWAY_TIMEOUT,
+ common.TOO_MANY_REQUESTS,
+ http.client.INTERNAL_SERVER_ERROR,
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.BAD_GATEWAY,
+ common.TOO_MANY_REQUESTS,
+ )
+ responses = [_make_response(status_code) for status_code in status_codes]
+
+ def raise_response():
+ raise common.InvalidResponse(responses.pop(0))
+
+ func = mock.Mock(side_effect=raise_response)
+
+ retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0)
+ try:
+ _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+ except common.InvalidResponse as e:
+ ret_val = e.response
+
+ assert ret_val.status_code == status_codes[-1]
+ assert status_codes[-1] in common.RETRYABLE
+
+ assert func.call_count == 7
+ assert func.mock_calls == [mock.call()] * 7
+
+ assert randint_mock.call_count == 7
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7
+
+ assert sleep_mock.call_count == 6
+ sleep_mock.assert_any_call(1.875)
+ sleep_mock.assert_any_call(2.0)
+ sleep_mock.assert_any_call(4.375)
+ sleep_mock.assert_any_call(8.5)
+ sleep_mock.assert_any_call(16.5)
+ sleep_mock.assert_any_call(32.25)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_retry_exceeds_max_retries(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125]
+
+ status_codes = (
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.GATEWAY_TIMEOUT,
+ common.TOO_MANY_REQUESTS,
+ http.client.INTERNAL_SERVER_ERROR,
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.BAD_GATEWAY,
+ common.TOO_MANY_REQUESTS,
+ )
+ responses = [_make_response(status_code) for status_code in status_codes]
+
+ def raise_response():
+ raise common.InvalidResponse(responses.pop(0))
+
+ func = mock.Mock(side_effect=raise_response)
+
+ retry_strategy = common.RetryStrategy(max_retries=6)
+ try:
+ _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+ except common.InvalidResponse as e:
+ ret_val = e.response
+
+ assert ret_val.status_code == status_codes[-1]
+ assert status_codes[-1] in common.RETRYABLE
+
+ assert func.call_count == 7
+ assert func.mock_calls == [mock.call()] * 7
+
+ assert randint_mock.call_count == 7
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7
+
+ assert sleep_mock.call_count == 6
+ sleep_mock.assert_any_call(1.875)
+ sleep_mock.assert_any_call(2.0)
+ sleep_mock.assert_any_call(4.375)
+ sleep_mock.assert_any_call(8.5)
+ sleep_mock.assert_any_call(16.5)
+ sleep_mock.assert_any_call(32.25)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_retry_zero_max_retries(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [875, 0, 375]
+
+ status_codes = (
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.GATEWAY_TIMEOUT,
+ common.TOO_MANY_REQUESTS,
+ )
+ responses = [_make_response(status_code) for status_code in status_codes]
+
+ def raise_response():
+ raise common.InvalidResponse(responses.pop(0))
+
+ func = mock.Mock(side_effect=raise_response)
+
+ retry_strategy = common.RetryStrategy(max_retries=0)
+ try:
+ _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+ except common.InvalidResponse as e:
+ ret_val = e.response
+
+ assert func.call_count == 1
+ assert func.mock_calls == [mock.call()] * 1
+ assert ret_val.status_code == status_codes[0]
+
+ assert randint_mock.call_count == 1
+ assert sleep_mock.call_count == 0
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ def test_retry_exceeded_reraises_connection_error(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125]
+
+ responses = [requests.exceptions.ConnectionError] * 7
+ func = mock.Mock(side_effect=responses, spec=[])
+
+ retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0)
+ with pytest.raises(requests.exceptions.ConnectionError):
+ _request_helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+
+ assert func.call_count == 7
+ assert func.mock_calls == [mock.call()] * 7
+
+ assert randint_mock.call_count == 7
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7
+
+ assert sleep_mock.call_count == 6
+ sleep_mock.assert_any_call(1.875)
+ sleep_mock.assert_any_call(2.0)
+ sleep_mock.assert_any_call(4.375)
+ sleep_mock.assert_any_call(8.5)
+ sleep_mock.assert_any_call(16.5)
+ sleep_mock.assert_any_call(32.25)
diff --git a/packages/google-resumable-media/tests/unit/requests/test_download.py b/packages/google-resumable-media/tests/unit/requests/test_download.py
new file mode 100644
index 000000000000..57cb4ed29d7c
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/requests/test_download.py
@@ -0,0 +1,1418 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+
+from unittest import mock
+import pytest # type: ignore
+
+from google.resumable_media import common
+from google.resumable_media import _helpers
+from google.resumable_media.requests import download as download_mod
+from google.resumable_media.requests import _request_helpers
+
+
+URL_PREFIX = "https://www.googleapis.com/download/storage/v1/b/{BUCKET}/o/"
+EXAMPLE_URL = URL_PREFIX + "{OBJECT}?alt=media"
+EXPECTED_TIMEOUT = (61, 60)
+
+
+class TestDownload(object):
+ def test__write_to_stream_no_hash_check(self):
+ stream = io.BytesIO()
+ download = download_mod.Download(EXAMPLE_URL, stream=stream)
+
+ chunk1 = b"right now, "
+ chunk2 = b"but a little later"
+ response = _mock_response(chunks=[chunk1, chunk2], headers={})
+
+ ret_val = download._write_to_stream(response)
+ assert ret_val is None
+
+ assert stream.getvalue() == chunk1 + chunk2
+ assert download._bytes_downloaded == len(chunk1 + chunk2)
+
+ # Check mocks.
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.iter_content.assert_called_once_with(
+ chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ def test__write_to_stream_with_hash_check_success(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.Download(EXAMPLE_URL, stream=stream, checksum=checksum)
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ header_value = "crc32c=qmNCyg==,md5=fPAJHnnoi/+NadyNxT2c2w=="
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ ret_val = download._write_to_stream(response)
+ assert ret_val is None
+
+ assert stream.getvalue() == chunk1 + chunk2 + chunk3
+ assert download._bytes_downloaded == len(chunk1 + chunk2 + chunk3)
+ assert download._checksum_object is not None
+
+ # Check mocks.
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.iter_content.assert_called_once_with(
+ chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__write_to_stream_with_hash_check_fail(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.Download(EXAMPLE_URL, stream=stream, checksum=checksum)
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ with pytest.raises(common.DataCorruption) as exc_info:
+ download._write_to_stream(response)
+
+ assert not download.finished
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 1
+ if checksum == "md5":
+ good_checksum = "fPAJHnnoi/+NadyNxT2c2w=="
+ else:
+ good_checksum = "qmNCyg=="
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ EXAMPLE_URL, bad_checksum, good_checksum, checksum_type=checksum.upper()
+ )
+ assert msg in error.args[0]
+ assert (
+ f"The download request read {download._bytes_downloaded} bytes of data."
+ in error.args[0]
+ )
+
+ # Check mocks.
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.iter_content.assert_called_once_with(
+ chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__write_to_stream_no_checksum_validation_for_partial_response(
+ self, checksum
+ ):
+ stream = io.BytesIO()
+ download = download_mod.Download(EXAMPLE_URL, stream=stream, checksum=checksum)
+
+ chunk1 = b"first chunk"
+ response = _mock_response(
+ status_code=http.client.PARTIAL_CONTENT, chunks=[chunk1]
+ )
+
+ # Make sure that the checksum is not validated.
+ with mock.patch(
+ "google.resumable_media._helpers.prepare_checksum_digest",
+ return_value=None,
+ ) as prepare_checksum_digest:
+ download._write_to_stream(response)
+ assert not prepare_checksum_digest.called
+
+ assert not download.finished
+
+ # Check mocks.
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.iter_content.assert_called_once_with(
+ chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
+ )
+
+ def test__write_to_stream_with_invalid_checksum_type(self):
+ BAD_CHECKSUM_TYPE = "badsum"
+
+ stream = io.BytesIO()
+ download = download_mod.Download(
+ EXAMPLE_URL, stream=stream, checksum=BAD_CHECKSUM_TYPE
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ with pytest.raises(ValueError) as exc_info:
+ download._write_to_stream(response)
+
+ assert not download.finished
+
+ error = exc_info.value
+ assert error.args[0] == "checksum must be ``'md5'``, ``'crc32c'`` or ``None``"
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__write_to_stream_incomplete_read(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.Download(EXAMPLE_URL, stream=stream, checksum=checksum)
+
+ chunk1 = b"first chunk"
+ mock_full_content_length = len(chunk1) + 123
+ headers = {"x-goog-stored-content-length": mock_full_content_length}
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers[_helpers._HASH_HEADER] = header_value
+ response = _mock_response(chunks=[chunk1], headers=headers)
+
+ with pytest.raises(ConnectionError) as exc_info:
+ download._write_to_stream(response)
+
+ assert not download.finished
+ error = exc_info.value
+ assert (
+ f"The download request read {download._bytes_downloaded} bytes of data."
+ in error.args[0]
+ )
+
+ def _consume_helper(
+ self,
+ stream=None,
+ end=65536,
+ headers=None,
+ chunks=(),
+ response_headers=None,
+ checksum="md5",
+ timeout=None,
+ ):
+ download = download_mod.Download(
+ EXAMPLE_URL, stream=stream, end=end, headers=headers, checksum=checksum
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_response(
+ chunks=chunks, headers=response_headers
+ )
+
+ assert not download.finished
+
+ if timeout is not None:
+ ret_val = download.consume(transport, timeout=timeout)
+ else:
+ ret_val = download.consume(transport)
+
+ assert ret_val is transport.request.return_value
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT if timeout is None else timeout,
+ }
+ if chunks:
+ assert stream is not None
+ called_kwargs["stream"] = True
+
+ transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
+
+ range_bytes = "bytes={:d}-{:d}".format(0, end)
+ assert download._headers["range"] == range_bytes
+ assert download.finished
+
+ return transport
+
+ def test_consume(self):
+ self._consume_helper()
+
+ def test_consume_with_custom_timeout(self):
+ self._consume_helper(timeout=14.7)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ def test_consume_with_stream(self, checksum):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ transport = self._consume_helper(
+ stream=stream, chunks=chunks, checksum=checksum
+ )
+
+ assert stream.getvalue() == b"".join(chunks)
+
+ # Check mocks.
+ response = transport.request.return_value
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.iter_content.assert_called_once_with(
+ chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test_consume_with_stream_hash_check_success(self, checksum):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ header_value = "crc32c=UNIQxg==,md5=JvS1wjMvfbCXgEGeaJJLDQ=="
+ headers = {_helpers._HASH_HEADER: header_value}
+ transport = self._consume_helper(
+ stream=stream, chunks=chunks, response_headers=headers, checksum=checksum
+ )
+
+ assert stream.getvalue() == b"".join(chunks)
+
+ # Check mocks.
+ response = transport.request.return_value
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.iter_content.assert_called_once_with(
+ chunk_size=_request_helpers._SINGLE_GET_CHUNK_SIZE, decode_unicode=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test_consume_with_stream_hash_check_fail(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.Download(EXAMPLE_URL, stream=stream, checksum=checksum)
+
+ chunks = (b"zero zero", b"niner tango")
+ bad_checksum = "anVzdCBub3QgdGhpcyAxLA=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_response(chunks=chunks, headers=headers)
+
+ assert not download.finished
+ with pytest.raises(common.DataCorruption) as exc_info:
+ download.consume(transport)
+
+ assert stream.getvalue() == b"".join(chunks)
+ assert download.finished
+ assert download._headers == {}
+
+ error = exc_info.value
+ assert error.response is transport.request.return_value
+ assert len(error.args) == 1
+ if checksum == "md5":
+ good_checksum = "1A/dxEpys717C6FH7FIWDw=="
+ else:
+ good_checksum = "GvNZlg=="
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ EXAMPLE_URL, bad_checksum, good_checksum, checksum_type=checksum.upper()
+ )
+ assert msg in error.args[0]
+ assert (
+ f"The download request read {download._bytes_downloaded} bytes of data."
+ in error.args[0]
+ )
+
+ # Check mocks.
+ transport.request.assert_called_once_with(
+ "GET",
+ EXAMPLE_URL,
+ data=None,
+ headers={},
+ stream=True,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ def test_consume_with_headers(self):
+ headers = {} # Empty headers
+ end = 16383
+ self._consume_helper(end=end, headers=headers)
+ range_bytes = "bytes={:d}-{:d}".format(0, end)
+ # Make sure the headers have been modified.
+ assert headers == {"range": range_bytes}
+
+ def test_consume_gets_generation_from_url(self):
+ GENERATION_VALUE = 1641590104888641
+ url = EXAMPLE_URL + f"&generation={GENERATION_VALUE}"
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+
+ download = download_mod.Download(
+ url, stream=stream, end=65536, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_response(chunks=chunks, headers=None)
+
+ assert not download.finished
+ assert download._object_generation is None
+
+ ret_val = download.consume(transport)
+
+ assert download._object_generation == GENERATION_VALUE
+ assert ret_val is transport.request.return_value
+ assert stream.getvalue() == b"".join(chunks)
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", url, **called_kwargs)
+
+ def test_consume_gets_generation_from_headers(self):
+ GENERATION_VALUE = 1641590104888641
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+
+ download = download_mod.Download(
+ EXAMPLE_URL, stream=stream, end=65536, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+ headers = {_helpers._GENERATION_HEADER: GENERATION_VALUE}
+ transport.request.return_value = _mock_response(chunks=chunks, headers=headers)
+
+ assert not download.finished
+ assert download._object_generation is None
+
+ ret_val = download.consume(transport)
+
+ assert download._object_generation == GENERATION_VALUE
+ assert ret_val is transport.request.return_value
+ assert stream.getvalue() == b"".join(chunks)
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
+
+ def test_consume_w_object_generation(self):
+ GENERATION_VALUE = 1641590104888641
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ end = 65536
+
+ download = download_mod.Download(
+ EXAMPLE_URL, stream=stream, end=end, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_response(chunks=chunks, headers=None)
+
+ assert download._object_generation is None
+
+ # Mock a retry operation with object generation retrieved and bytes already downloaded in the stream
+ download._object_generation = GENERATION_VALUE
+ offset = 256
+ download._bytes_downloaded = offset
+ download.consume(transport)
+
+ expected_url = EXAMPLE_URL + f"&generation={GENERATION_VALUE}"
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", expected_url, **called_kwargs)
+ range_bytes = "bytes={:d}-{:d}".format(offset, end)
+ assert download._headers["range"] == range_bytes
+
+ def test_consume_w_bytes_downloaded(self):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ end = 65536
+
+ download = download_mod.Download(
+ EXAMPLE_URL, stream=stream, end=end, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_response(chunks=chunks, headers=None)
+
+ assert download._bytes_downloaded == 0
+
+ # Mock a retry operation with bytes already downloaded in the stream and checksum stored
+ offset = 256
+ download._bytes_downloaded = offset
+ download._expected_checksum = None
+ download._checksum_object = _helpers._DoNothingHash()
+ download.consume(transport)
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
+ range_bytes = "bytes={:d}-{:d}".format(offset, end)
+ assert download._headers["range"] == range_bytes
+
+ def test_consume_w_bytes_downloaded_range_read(self):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ start = 1024
+ end = 65536
+
+ download = download_mod.Download(
+ EXAMPLE_URL,
+ stream=stream,
+ start=start,
+ end=end,
+ headers=None,
+ checksum="md5",
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_response(chunks=chunks, headers=None)
+
+ assert download._bytes_downloaded == 0
+
+ # Mock a retry operation with bytes already downloaded in the stream and checksum stored
+ offset = 256
+ download._bytes_downloaded = offset
+ download._expected_checksum = None
+ download._checksum_object = _helpers._DoNothingHash()
+ download.consume(transport)
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
+ range_bytes = "bytes={:d}-{:d}".format(offset + start, end)
+ assert download._headers["range"] == range_bytes
+
+ def test_consume_gzip_reset_stream_w_bytes_downloaded(self):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ end = 65536
+
+ download = download_mod.Download(
+ EXAMPLE_URL, stream=stream, end=end, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+
+ # Mock a decompressive transcoding retry operation with bytes already downloaded in the stream
+ headers = {_helpers._STORED_CONTENT_ENCODING_HEADER: "gzip"}
+ transport.request.return_value = _mock_response(chunks=chunks, headers=headers)
+ offset = 16
+ download._bytes_downloaded = offset
+ download.consume(transport)
+
+ assert stream.getvalue() == b"".join(chunks)
+ assert download._bytes_downloaded == len(b"".join(chunks))
+
+ def test_consume_gzip_reset_stream_error(self):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ end = 65536
+
+ download = download_mod.Download(
+ EXAMPLE_URL, stream=stream, end=end, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+
+ # Mock a stream seek error while resuming a decompressive transcoding download
+ stream.seek = mock.Mock(side_effect=OSError("mock stream seek error"))
+ headers = {_helpers._STORED_CONTENT_ENCODING_HEADER: "gzip"}
+ transport.request.return_value = _mock_response(chunks=chunks, headers=headers)
+ offset = 16
+ download._bytes_downloaded = offset
+ with pytest.raises(Exception):
+ download.consume(transport)
+
+
+class TestRawDownload(object):
+ def test__write_to_stream_no_hash_check(self):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(EXAMPLE_URL, stream=stream)
+
+ chunk1 = b"right now, "
+ chunk2 = b"but a little later"
+ response = _mock_raw_response(chunks=[chunk1, chunk2], headers={})
+
+ ret_val = download._write_to_stream(response)
+ assert ret_val is None
+
+ assert stream.getvalue() == chunk1 + chunk2
+ assert download._bytes_downloaded == len(chunk1 + chunk2)
+
+ # Check mocks.
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.raw.stream.assert_called_once_with(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ def test__write_to_stream_with_hash_check_success(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ header_value = "crc32c=qmNCyg==,md5=fPAJHnnoi/+NadyNxT2c2w=="
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ ret_val = download._write_to_stream(response)
+ assert ret_val is None
+
+ assert stream.getvalue() == chunk1 + chunk2 + chunk3
+ assert download._bytes_downloaded == len(chunk1 + chunk2 + chunk3)
+ assert download._checksum_object is not None
+
+ # Check mocks.
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.raw.stream.assert_called_once_with(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__write_to_stream_with_hash_check_fail(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ with pytest.raises(common.DataCorruption) as exc_info:
+ download._write_to_stream(response)
+
+ assert not download.finished
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 1
+ if checksum == "md5":
+ good_checksum = "fPAJHnnoi/+NadyNxT2c2w=="
+ else:
+ good_checksum = "qmNCyg=="
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ EXAMPLE_URL, bad_checksum, good_checksum, checksum_type=checksum.upper()
+ )
+ assert msg in error.args[0]
+ assert (
+ f"The download request read {download._bytes_downloaded} bytes of data."
+ in error.args[0]
+ )
+
+ # Check mocks.
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.raw.stream.assert_called_once_with(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
+ )
+
+ def test__write_to_stream_with_invalid_checksum_type(self):
+ BAD_CHECKSUM_TYPE = "badsum"
+
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, checksum=BAD_CHECKSUM_TYPE
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ with pytest.raises(ValueError) as exc_info:
+ download._write_to_stream(response)
+
+ assert not download.finished
+
+ error = exc_info.value
+ assert error.args[0] == "checksum must be ``'md5'``, ``'crc32c'`` or ``None``"
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__write_to_stream_incomplete_read(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunk1 = b"first chunk"
+ mock_full_content_length = len(chunk1) + 123
+ headers = {"x-goog-stored-content-length": mock_full_content_length}
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers[_helpers._HASH_HEADER] = header_value
+ response = _mock_raw_response(chunks=[chunk1], headers=headers)
+
+ with pytest.raises(ConnectionError) as exc_info:
+ download._write_to_stream(response)
+
+ assert not download.finished
+ error = exc_info.value
+ assert (
+ f"The download request read {download._bytes_downloaded} bytes of data."
+ in error.args[0]
+ )
+
+ def _consume_helper(
+ self,
+ stream=None,
+ end=65536,
+ headers=None,
+ chunks=(),
+ response_headers=None,
+ checksum=None,
+ timeout=None,
+ ):
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, end=end, headers=headers, checksum=checksum
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_raw_response(
+ chunks=chunks, headers=response_headers
+ )
+
+ assert not download.finished
+
+ if timeout is not None:
+ ret_val = download.consume(transport, timeout=timeout)
+ else:
+ ret_val = download.consume(transport)
+
+ assert ret_val is transport.request.return_value
+
+ if chunks:
+ assert stream is not None
+ transport.request.assert_called_once_with(
+ "GET",
+ EXAMPLE_URL,
+ data=None,
+ headers=download._headers,
+ stream=True,
+ timeout=EXPECTED_TIMEOUT if timeout is None else timeout,
+ )
+
+ range_bytes = "bytes={:d}-{:d}".format(0, end)
+ assert download._headers["range"] == range_bytes
+ assert download.finished
+
+ return transport
+
+ def test_consume(self):
+ self._consume_helper()
+
+ def test_consume_with_custom_timeout(self):
+ self._consume_helper(timeout=14.7)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ def test_consume_with_stream(self, checksum):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ transport = self._consume_helper(
+ stream=stream, chunks=chunks, checksum=checksum
+ )
+
+ assert stream.getvalue() == b"".join(chunks)
+
+ # Check mocks.
+ response = transport.request.return_value
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.raw.stream.assert_called_once_with(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test_consume_with_stream_hash_check_success(self, checksum):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ header_value = "crc32c=UNIQxg==,md5=JvS1wjMvfbCXgEGeaJJLDQ=="
+ headers = {_helpers._HASH_HEADER: header_value}
+ transport = self._consume_helper(
+ stream=stream, chunks=chunks, response_headers=headers, checksum=checksum
+ )
+
+ assert stream.getvalue() == b"".join(chunks)
+
+ # Check mocks.
+ response = transport.request.return_value
+ response.__enter__.assert_called_once_with()
+ response.__exit__.assert_called_once_with(None, None, None)
+ response.raw.stream.assert_called_once_with(
+ _request_helpers._SINGLE_GET_CHUNK_SIZE, decode_content=False
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test_consume_with_stream_hash_check_fail(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunks = (b"zero zero", b"niner tango")
+ bad_checksum = "anVzdCBub3QgdGhpcyAxLA=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_raw_response(
+ chunks=chunks, headers=headers
+ )
+
+ assert not download.finished
+ with pytest.raises(common.DataCorruption) as exc_info:
+ download.consume(transport)
+
+ assert stream.getvalue() == b"".join(chunks)
+ assert download.finished
+ assert download._headers == {}
+
+ error = exc_info.value
+ assert error.response is transport.request.return_value
+ assert len(error.args) == 1
+ if checksum == "md5":
+ good_checksum = "1A/dxEpys717C6FH7FIWDw=="
+ else:
+ good_checksum = "GvNZlg=="
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ EXAMPLE_URL, bad_checksum, good_checksum, checksum_type=checksum.upper()
+ )
+ assert msg in error.args[0]
+ assert (
+ f"The download request read {download._bytes_downloaded} bytes of data."
+ in error.args[0]
+ )
+
+ # Check mocks.
+ transport.request.assert_called_once_with(
+ "GET",
+ EXAMPLE_URL,
+ data=None,
+ headers={},
+ stream=True,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ def test_consume_with_headers(self):
+ headers = {} # Empty headers
+ end = 16383
+ self._consume_helper(end=end, headers=headers)
+ range_bytes = "bytes={:d}-{:d}".format(0, end)
+ # Make sure the headers have been modified.
+ assert headers == {"range": range_bytes}
+
+ def test_consume_gets_generation_from_url(self):
+ GENERATION_VALUE = 1641590104888641
+ url = EXAMPLE_URL + f"&generation={GENERATION_VALUE}"
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+
+ download = download_mod.RawDownload(
+ url, stream=stream, end=65536, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_raw_response(chunks=chunks, headers=None)
+
+ assert not download.finished
+ assert download._object_generation is None
+
+ ret_val = download.consume(transport)
+
+ assert download._object_generation == GENERATION_VALUE
+ assert ret_val is transport.request.return_value
+ assert stream.getvalue() == b"".join(chunks)
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", url, **called_kwargs)
+
+ def test_consume_gets_generation_from_headers(self):
+ GENERATION_VALUE = 1641590104888641
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, end=65536, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+ headers = {_helpers._GENERATION_HEADER: GENERATION_VALUE}
+ transport.request.return_value = _mock_raw_response(
+ chunks=chunks, headers=headers
+ )
+
+ assert not download.finished
+ assert download._object_generation is None
+
+ ret_val = download.consume(transport)
+
+ assert download._object_generation == GENERATION_VALUE
+ assert ret_val is transport.request.return_value
+ assert stream.getvalue() == b"".join(chunks)
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
+
+ def test_consume_w_object_generation(self):
+ GENERATION_VALUE = 1641590104888641
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ end = 65536
+
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, end=end, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_raw_response(chunks=chunks, headers=None)
+
+ assert download._object_generation is None
+
+ # Mock a retry operation with object generation retrieved and bytes already downloaded in the stream
+ download._object_generation = GENERATION_VALUE
+ offset = 256
+ download._bytes_downloaded = offset
+ download.consume(transport)
+
+ expected_url = EXAMPLE_URL + f"&generation={GENERATION_VALUE}"
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", expected_url, **called_kwargs)
+ range_bytes = "bytes={:d}-{:d}".format(offset, end)
+ assert download._headers["range"] == range_bytes
+
+ def test_consume_w_bytes_downloaded(self):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ end = 65536
+
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, end=end, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_raw_response(chunks=chunks, headers=None)
+
+ assert download._bytes_downloaded == 0
+
+ # Mock a retry operation with bytes already downloaded in the stream and checksum stored
+ offset = 256
+ download._bytes_downloaded = offset
+ download._expected_checksum = None
+ download._checksum_object = _helpers._DoNothingHash()
+ download.consume(transport)
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
+ range_bytes = "bytes={:d}-{:d}".format(offset, end)
+ assert download._headers["range"] == range_bytes
+
+ def test_consume_w_bytes_downloaded_range_read(self):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ start = 1024
+ end = 65536
+
+ download = download_mod.RawDownload(
+ EXAMPLE_URL,
+ stream=stream,
+ start=start,
+ end=end,
+ headers=None,
+ checksum="md5",
+ )
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _mock_raw_response(chunks=chunks, headers=None)
+
+ assert download._bytes_downloaded == 0
+
+ # Mock a retry operation with bytes already downloaded in the stream and checksum stored
+ offset = 256
+ download._bytes_downloaded = offset
+ download._expected_checksum = None
+ download._checksum_object = _helpers._DoNothingHash()
+ download.consume(transport)
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT,
+ "stream": True,
+ }
+ transport.request.assert_called_once_with("GET", EXAMPLE_URL, **called_kwargs)
+ range_bytes = "bytes={:d}-{:d}".format(start + offset, end)
+ assert download._headers["range"] == range_bytes
+
+ def test_consume_gzip_reset_stream_w_bytes_downloaded(self):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ end = 65536
+
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, end=end, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+
+ # Mock a decompressive transcoding retry operation with bytes already downloaded in the stream
+ headers = {_helpers._STORED_CONTENT_ENCODING_HEADER: "gzip"}
+ transport.request.return_value = _mock_raw_response(
+ chunks=chunks, headers=headers
+ )
+ offset = 16
+ download._bytes_downloaded = offset
+ download.consume(transport)
+
+ assert stream.getvalue() == b"".join(chunks)
+ assert download._bytes_downloaded == len(b"".join(chunks))
+
+ def test_consume_gzip_reset_stream_error(self):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ end = 65536
+
+ download = download_mod.RawDownload(
+ EXAMPLE_URL, stream=stream, end=end, headers=None, checksum="md5"
+ )
+ transport = mock.Mock(spec=["request"])
+
+ # Mock a stream seek error while resuming a decompressive transcoding download
+ stream.seek = mock.Mock(side_effect=OSError("mock stream seek error"))
+ headers = {_helpers._STORED_CONTENT_ENCODING_HEADER: "gzip"}
+ transport.request.return_value = _mock_raw_response(
+ chunks=chunks, headers=headers
+ )
+ offset = 16
+ download._bytes_downloaded = offset
+ with pytest.raises(Exception):
+ download.consume(transport)
+
+
+class TestChunkedDownload(object):
+ @staticmethod
+ def _response_content_range(start_byte, end_byte, total_bytes):
+ return "bytes {:d}-{:d}/{:d}".format(start_byte, end_byte, total_bytes)
+
+ def _response_headers(self, start_byte, end_byte, total_bytes):
+ content_length = end_byte - start_byte + 1
+ resp_range = self._response_content_range(start_byte, end_byte, total_bytes)
+ return {
+ "content-length": "{:d}".format(content_length),
+ "content-range": resp_range,
+ }
+
+ def _mock_response(
+ self, start_byte, end_byte, total_bytes, content=None, status_code=None
+ ):
+ response_headers = self._response_headers(start_byte, end_byte, total_bytes)
+ return mock.Mock(
+ content=content,
+ headers=response_headers,
+ status_code=status_code,
+ spec=["content", "headers", "status_code"],
+ )
+
+ def test_consume_next_chunk_already_finished(self):
+ download = download_mod.ChunkedDownload(EXAMPLE_URL, 512, None)
+ download._finished = True
+ with pytest.raises(ValueError):
+ download.consume_next_chunk(None)
+
+ def _mock_transport(self, start, chunk_size, total_bytes, content=b""):
+ transport = mock.Mock(spec=["request"])
+ assert len(content) == chunk_size
+ transport.request.return_value = self._mock_response(
+ start,
+ start + chunk_size - 1,
+ total_bytes,
+ content=content,
+ status_code=int(http.client.OK),
+ )
+
+ return transport
+
+ def test_consume_next_chunk(self):
+ start = 1536
+ stream = io.BytesIO()
+ data = b"Just one chunk."
+ chunk_size = len(data)
+ download = download_mod.ChunkedDownload(
+ EXAMPLE_URL, chunk_size, stream, start=start
+ )
+ total_bytes = 16384
+ transport = self._mock_transport(start, chunk_size, total_bytes, content=data)
+
+ # Verify the internal state before consuming a chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually consume the chunk and check the output.
+ ret_val = download.consume_next_chunk(transport)
+ assert ret_val is transport.request.return_value
+ range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1)
+ download_headers = {"range": range_bytes}
+ transport.request.assert_called_once_with(
+ "GET",
+ EXAMPLE_URL,
+ data=None,
+ headers=download_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+ assert stream.getvalue() == data
+ # Go back and check the internal state after consuming the chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == chunk_size
+ assert download.total_bytes == total_bytes
+
+ def test_consume_next_chunk_with_custom_timeout(self):
+ start = 1536
+ stream = io.BytesIO()
+ data = b"Just one chunk."
+ chunk_size = len(data)
+ download = download_mod.ChunkedDownload(
+ EXAMPLE_URL, chunk_size, stream, start=start
+ )
+ total_bytes = 16384
+ transport = self._mock_transport(start, chunk_size, total_bytes, content=data)
+
+ # Actually consume the chunk and check the output.
+ download.consume_next_chunk(transport, timeout=14.7)
+
+ range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1)
+ download_headers = {"range": range_bytes}
+ transport.request.assert_called_once_with(
+ "GET",
+ EXAMPLE_URL,
+ data=None,
+ headers=download_headers,
+ timeout=14.7,
+ )
+
+
+class TestRawChunkedDownload(object):
+ @staticmethod
+ def _response_content_range(start_byte, end_byte, total_bytes):
+ return "bytes {:d}-{:d}/{:d}".format(start_byte, end_byte, total_bytes)
+
+ def _response_headers(self, start_byte, end_byte, total_bytes):
+ content_length = end_byte - start_byte + 1
+ resp_range = self._response_content_range(start_byte, end_byte, total_bytes)
+ return {
+ "content-length": "{:d}".format(content_length),
+ "content-range": resp_range,
+ }
+
+ def _mock_response(
+ self, start_byte, end_byte, total_bytes, content=None, status_code=None
+ ):
+ response_headers = self._response_headers(start_byte, end_byte, total_bytes)
+ return mock.Mock(
+ _content=content,
+ headers=response_headers,
+ status_code=status_code,
+ spec=["_content", "headers", "status_code"],
+ )
+
+ def test_consume_next_chunk_already_finished(self):
+ download = download_mod.RawChunkedDownload(EXAMPLE_URL, 512, None)
+ download._finished = True
+ with pytest.raises(ValueError):
+ download.consume_next_chunk(None)
+
+ def _mock_transport(self, start, chunk_size, total_bytes, content=b""):
+ transport = mock.Mock(spec=["request"])
+ assert len(content) == chunk_size
+ transport.request.return_value = self._mock_response(
+ start,
+ start + chunk_size - 1,
+ total_bytes,
+ content=content,
+ status_code=int(http.client.OK),
+ )
+
+ return transport
+
+ def test_consume_next_chunk(self):
+ start = 1536
+ stream = io.BytesIO()
+ data = b"Just one chunk."
+ chunk_size = len(data)
+ download = download_mod.RawChunkedDownload(
+ EXAMPLE_URL, chunk_size, stream, start=start
+ )
+ total_bytes = 16384
+ transport = self._mock_transport(start, chunk_size, total_bytes, content=data)
+
+ # Verify the internal state before consuming a chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually consume the chunk and check the output.
+ ret_val = download.consume_next_chunk(transport)
+ assert ret_val is transport.request.return_value
+ range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1)
+ download_headers = {"range": range_bytes}
+ transport.request.assert_called_once_with(
+ "GET",
+ EXAMPLE_URL,
+ data=None,
+ headers=download_headers,
+ stream=True,
+ timeout=EXPECTED_TIMEOUT,
+ )
+ assert stream.getvalue() == data
+ # Go back and check the internal state after consuming the chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == chunk_size
+ assert download.total_bytes == total_bytes
+
+ def test_consume_next_chunk_with_custom_timeout(self):
+ start = 1536
+ stream = io.BytesIO()
+ data = b"Just one chunk."
+ chunk_size = len(data)
+ download = download_mod.RawChunkedDownload(
+ EXAMPLE_URL, chunk_size, stream, start=start
+ )
+ total_bytes = 16384
+ transport = self._mock_transport(start, chunk_size, total_bytes, content=data)
+
+ # Actually consume the chunk and check the output.
+ download.consume_next_chunk(transport, timeout=14.7)
+
+ range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1)
+ download_headers = {"range": range_bytes}
+ transport.request.assert_called_once_with(
+ "GET",
+ EXAMPLE_URL,
+ data=None,
+ headers=download_headers,
+ stream=True,
+ timeout=14.7,
+ )
+ assert stream.getvalue() == data
+ # Go back and check the internal state after consuming the chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == chunk_size
+ assert download.total_bytes == total_bytes
+
+
+class Test__add_decoder(object):
+ def test_non_gzipped(self):
+ response_raw = mock.Mock(headers={}, spec=["headers"])
+ md5_hash = download_mod._add_decoder(response_raw, mock.sentinel.md5_hash)
+
+ assert md5_hash is mock.sentinel.md5_hash
+
+ def test_gzipped(self):
+ headers = {"content-encoding": "gzip"}
+ response_raw = mock.Mock(headers=headers, spec=["headers", "_decoder"])
+ md5_hash = download_mod._add_decoder(response_raw, mock.sentinel.md5_hash)
+
+ assert md5_hash is not mock.sentinel.md5_hash
+ assert isinstance(md5_hash, _helpers._DoNothingHash)
+ assert isinstance(response_raw._decoder, download_mod._GzipDecoder)
+ assert response_raw._decoder._checksum is mock.sentinel.md5_hash
+
+ def test_brotli(self):
+ headers = {"content-encoding": "br"}
+ response_raw = mock.Mock(headers=headers, spec=["headers", "_decoder"])
+ md5_hash = download_mod._add_decoder(response_raw, mock.sentinel.md5_hash)
+
+ assert md5_hash is not mock.sentinel.md5_hash
+ assert isinstance(md5_hash, _helpers._DoNothingHash)
+ assert isinstance(response_raw._decoder, download_mod._BrotliDecoder)
+ assert response_raw._decoder._checksum is mock.sentinel.md5_hash
+ # Go ahead and exercise the flush method, added only for completion
+ response_raw._decoder.flush()
+
+
+class Test_GzipDecoder(object):
+ def test_constructor(self):
+ decoder = download_mod._GzipDecoder(mock.sentinel.md5_hash)
+ assert decoder._checksum is mock.sentinel.md5_hash
+
+ def test_decompress(self):
+ md5_hash = mock.Mock(spec=["update"])
+ decoder = download_mod._GzipDecoder(md5_hash)
+
+ data = b"\x1f\x8b\x08\x08"
+ result = decoder.decompress(data)
+
+ assert result == b""
+ md5_hash.update.assert_called_once_with(data)
+
+ def test_decompress_with_max_length(self):
+ md5_hash = mock.Mock(spec=["update"])
+ decoder = download_mod._GzipDecoder(md5_hash)
+
+ with mock.patch.object(
+ type(decoder).__bases__[0], "decompress"
+ ) as mock_super_decompress:
+ mock_super_decompress.return_value = b"decompressed"
+ data = b"\x1f\x8b\x08\x08"
+ result = decoder.decompress(data, max_length=10)
+
+ assert result == b"decompressed"
+ md5_hash.update.assert_called_once_with(data)
+ mock_super_decompress.assert_called_once_with(data, max_length=10)
+
+ def test_decompress_with_max_length_fallback(self):
+ md5_hash = mock.Mock(spec=["update"])
+ decoder = download_mod._GzipDecoder(md5_hash)
+
+ with mock.patch.object(
+ type(decoder).__bases__[0],
+ "decompress",
+ side_effect=[TypeError, b"decompressed"],
+ ) as mock_super_decompress:
+ data = b"\x1f\x8b\x08\x08"
+ result = decoder.decompress(data, max_length=10)
+
+ assert result == b"decompressed"
+ md5_hash.update.assert_called_once_with(data)
+ assert mock_super_decompress.call_count == 2
+
+
+class Test_BrotliDecoder(object):
+ def test_constructor(self):
+ decoder = download_mod._BrotliDecoder(mock.sentinel.md5_hash)
+ assert decoder._checksum is mock.sentinel.md5_hash
+
+ def test_decompress(self):
+ md5_hash = mock.Mock(spec=["update"])
+ decoder = download_mod._BrotliDecoder(md5_hash)
+
+ data = b"\xc1\xf8I\xc0/\x83\xf3\xfa"
+ result = decoder.decompress(data)
+
+ assert result == b""
+ md5_hash.update.assert_called_once_with(data)
+
+ def test_decompress_with_max_length(self):
+ md5_hash = mock.Mock(spec=["update"])
+ decoder = download_mod._BrotliDecoder(md5_hash)
+
+ decoder._decoder = mock.Mock(spec=["decompress"])
+ decoder._decoder.decompress.return_value = b"decompressed"
+
+ data = b"compressed"
+ result = decoder.decompress(data, max_length=10)
+
+ assert result == b"decompressed"
+ md5_hash.update.assert_called_once_with(data)
+ decoder._decoder.decompress.assert_called_once_with(data, max_length=10)
+
+ def test_decompress_with_max_length_fallback(self):
+ md5_hash = mock.Mock(spec=["update"])
+ decoder = download_mod._BrotliDecoder(md5_hash)
+
+ decoder._decoder = mock.Mock(spec=["decompress"])
+ decoder._decoder.decompress.side_effect = [TypeError, b"decompressed"]
+
+ data = b"compressed"
+ result = decoder.decompress(data, max_length=10)
+
+ assert result == b"decompressed"
+ md5_hash.update.assert_called_once_with(data)
+ assert decoder._decoder.decompress.call_count == 2
+
+ def test_has_unconsumed_tail(self):
+ decoder = download_mod._BrotliDecoder(mock.sentinel.md5_hash)
+ decoder._decoder = mock.Mock(spec=["has_unconsumed_tail"])
+ decoder._decoder.has_unconsumed_tail = True
+ assert decoder.has_unconsumed_tail is True
+
+ def test_has_unconsumed_tail_fallback(self):
+ decoder = download_mod._BrotliDecoder(mock.sentinel.md5_hash)
+ decoder._decoder = mock.Mock(spec=[])
+ assert decoder.has_unconsumed_tail is False
+
+
+def _mock_response(status_code=http.client.OK, chunks=(), headers=None):
+ if headers is None:
+ headers = {}
+
+ if chunks:
+ mock_raw = mock.Mock(headers=headers, spec=["headers"])
+ response = mock.MagicMock(
+ headers=headers,
+ status_code=int(status_code),
+ raw=mock_raw,
+ spec=[
+ "__enter__",
+ "__exit__",
+ "iter_content",
+ "status_code",
+ "headers",
+ "raw",
+ ],
+ )
+ # i.e. context manager returns ``self``.
+ response.__enter__.return_value = response
+ response.__exit__.return_value = None
+ response.iter_content.return_value = iter(chunks)
+ return response
+ else:
+ return mock.Mock(
+ headers=headers,
+ status_code=int(status_code),
+ spec=["status_code", "headers"],
+ )
+
+
+def _mock_raw_response(status_code=http.client.OK, chunks=(), headers=None):
+ if headers is None:
+ headers = {}
+
+ mock_raw = mock.Mock(headers=headers, spec=["stream"])
+ mock_raw.stream.return_value = iter(chunks)
+ response = mock.MagicMock(
+ headers=headers,
+ status_code=int(status_code),
+ raw=mock_raw,
+ spec=[
+ "__enter__",
+ "__exit__",
+ "iter_content",
+ "status_code",
+ "headers",
+ "raw",
+ ],
+ )
+ # i.e. context manager returns ``self``.
+ response.__enter__.return_value = response
+ response.__exit__.return_value = None
+ return response
diff --git a/packages/google-resumable-media/tests/unit/requests/test_upload.py b/packages/google-resumable-media/tests/unit/requests/test_upload.py
new file mode 100644
index 000000000000..18bc06d91d4f
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/requests/test_upload.py
@@ -0,0 +1,406 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+import json
+import pytest # type: ignore
+import tempfile
+from unittest import mock
+
+import google.resumable_media.requests.upload as upload_mod
+
+
+URL_PREFIX = "https://www.googleapis.com/upload/storage/v1/b/{BUCKET}/o"
+SIMPLE_URL = URL_PREFIX + "?uploadType=media&name={OBJECT}"
+MULTIPART_URL = URL_PREFIX + "?uploadType=multipart"
+RESUMABLE_URL = URL_PREFIX + "?uploadType=resumable"
+ONE_MB = 1024 * 1024
+BASIC_CONTENT = "text/plain"
+JSON_TYPE = "application/json; charset=UTF-8"
+JSON_TYPE_LINE = b"content-type: application/json; charset=UTF-8\r\n"
+EXPECTED_TIMEOUT = (61, 60)
+EXAMPLE_XML_UPLOAD_URL = "https://test-project.storage.googleapis.com/test-bucket"
+EXAMPLE_XML_MPU_INITIATE_TEXT_TEMPLATE = """
+
+ travel-maps
+ paris.jpg
+ {upload_id}
+
+"""
+UPLOAD_ID = "VXBsb2FkIElEIGZvciBlbHZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"
+PARTS = {1: "39a59594290b0f9a30662a56d695b71d", 2: "00000000290b0f9a30662a56d695b71d"}
+FILE_DATA = b"testdata" * 128
+
+
+@pytest.fixture(scope="session")
+def filename():
+ with tempfile.NamedTemporaryFile() as f:
+ f.write(FILE_DATA)
+ f.flush()
+ yield f.name
+
+
+class TestSimpleUpload(object):
+ def test_transmit(self):
+ data = b"I have got a lovely bunch of coconuts."
+ content_type = BASIC_CONTENT
+ upload = upload_mod.SimpleUpload(SIMPLE_URL)
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response()
+ assert not upload.finished
+ ret_val = upload.transmit(transport, data, content_type)
+ assert ret_val is transport.request.return_value
+ upload_headers = {"content-type": content_type}
+ transport.request.assert_called_once_with(
+ "POST",
+ SIMPLE_URL,
+ data=data,
+ headers=upload_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+ assert upload.finished
+
+ def test_transmit_w_custom_timeout(self):
+ data = b"I have got a lovely bunch of coconuts."
+ content_type = BASIC_CONTENT
+ upload = upload_mod.SimpleUpload(SIMPLE_URL)
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response()
+
+ upload.transmit(transport, data, content_type, timeout=12.6)
+
+ expected_headers = {"content-type": content_type}
+ transport.request.assert_called_once_with(
+ "POST",
+ SIMPLE_URL,
+ data=data,
+ headers=expected_headers,
+ timeout=12.6,
+ )
+
+
+class TestMultipartUpload(object):
+ @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==4==")
+ def test_transmit(self, mock_get_boundary):
+ data = b"Mock data here and there."
+ metadata = {"Hey": "You", "Guys": "90909"}
+ content_type = BASIC_CONTENT
+ upload = upload_mod.MultipartUpload(MULTIPART_URL)
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response()
+ assert not upload.finished
+ ret_val = upload.transmit(transport, data, metadata, content_type)
+ assert ret_val is transport.request.return_value
+ expected_payload = (
+ b"--==4==\r\n"
+ + JSON_TYPE_LINE
+ + b"\r\n"
+ + json.dumps(metadata).encode("utf-8")
+ + b"\r\n"
+ + b"--==4==\r\n"
+ b"content-type: text/plain\r\n"
+ b"\r\n"
+ b"Mock data here and there.\r\n"
+ b"--==4==--"
+ )
+ multipart_type = b'multipart/related; boundary="==4=="'
+ upload_headers = {"content-type": multipart_type}
+ transport.request.assert_called_once_with(
+ "POST",
+ MULTIPART_URL,
+ data=expected_payload,
+ headers=upload_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+ assert upload.finished
+ mock_get_boundary.assert_called_once_with()
+
+ @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==4==")
+ def test_transmit_w_custom_timeout(self, mock_get_boundary):
+ data = b"Mock data here and there."
+ metadata = {"Hey": "You", "Guys": "90909"}
+ content_type = BASIC_CONTENT
+ upload = upload_mod.MultipartUpload(MULTIPART_URL)
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response()
+
+ upload.transmit(transport, data, metadata, content_type, timeout=12.6)
+
+ expected_payload = b"".join(
+ (
+ b"--==4==\r\n",
+ JSON_TYPE_LINE,
+ b"\r\n",
+ json.dumps(metadata).encode("utf-8"),
+ b"\r\n",
+ b"--==4==\r\n",
+ b"content-type: text/plain\r\n",
+ b"\r\n",
+ b"Mock data here and there.\r\n",
+ b"--==4==--",
+ )
+ )
+ multipart_type = b'multipart/related; boundary="==4=="'
+ upload_headers = {"content-type": multipart_type}
+
+ transport.request.assert_called_once_with(
+ "POST",
+ MULTIPART_URL,
+ data=expected_payload,
+ headers=upload_headers,
+ timeout=12.6,
+ )
+ assert upload.finished
+ mock_get_boundary.assert_called_once_with()
+
+
+class TestResumableUpload(object):
+ def test_initiate(self):
+ upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ data = b"Knock knock who is there"
+ stream = io.BytesIO(data)
+ metadata = {"name": "got-jokes.txt"}
+
+ transport = mock.Mock(spec=["request"])
+ location = ("http://test.invalid?upload_id=AACODBBBxuw9u3AA",)
+ response_headers = {"location": location}
+ post_response = _make_response(headers=response_headers)
+ transport.request.return_value = post_response
+ # Check resumable_url before.
+ assert upload._resumable_url is None
+ # Make request and check the return value (against the mock).
+ total_bytes = 100
+ assert total_bytes > len(data)
+ response = upload.initiate(
+ transport,
+ stream,
+ metadata,
+ BASIC_CONTENT,
+ total_bytes=total_bytes,
+ stream_final=False,
+ )
+ assert response is transport.request.return_value
+ # Check resumable_url after.
+ assert upload._resumable_url == location
+ # Make sure the mock was called as expected.
+ json_bytes = b'{"name": "got-jokes.txt"}'
+ expected_headers = {
+ "content-type": JSON_TYPE,
+ "x-upload-content-type": BASIC_CONTENT,
+ "x-upload-content-length": "{:d}".format(total_bytes),
+ }
+ transport.request.assert_called_once_with(
+ "POST",
+ RESUMABLE_URL,
+ data=json_bytes,
+ headers=expected_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ def test_initiate_w_custom_timeout(self):
+ upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ data = b"Knock knock who is there"
+ stream = io.BytesIO(data)
+ metadata = {"name": "got-jokes.txt"}
+
+ transport = mock.Mock(spec=["request"])
+ location = ("http://test.invalid?upload_id=AACODBBBxuw9u3AA",)
+ response_headers = {"location": location}
+ post_response = _make_response(headers=response_headers)
+ transport.request.return_value = post_response
+
+ upload.initiate(
+ transport,
+ stream,
+ metadata,
+ BASIC_CONTENT,
+ total_bytes=100,
+ timeout=12.6,
+ )
+
+ # Make sure timeout was passed to the transport
+ json_bytes = b'{"name": "got-jokes.txt"}'
+ expected_headers = {
+ "content-type": JSON_TYPE,
+ "x-upload-content-type": BASIC_CONTENT,
+ "x-upload-content-length": "{:d}".format(100),
+ }
+ transport.request.assert_called_once_with(
+ "POST",
+ RESUMABLE_URL,
+ data=json_bytes,
+ headers=expected_headers,
+ timeout=12.6,
+ )
+
+ @staticmethod
+ def _upload_in_flight(data, headers=None):
+ upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB, headers=headers)
+ upload._stream = io.BytesIO(data)
+ upload._content_type = BASIC_CONTENT
+ upload._total_bytes = len(data)
+ upload._resumable_url = "http://test.invalid?upload_id=not-none"
+ return upload
+
+ @staticmethod
+ def _chunk_mock(status_code, response_headers):
+ transport = mock.Mock(spec=["request"])
+ put_response = _make_response(status_code=status_code, headers=response_headers)
+ transport.request.return_value = put_response
+
+ return transport
+
+ def test_transmit_next_chunk(self):
+ data = b"This time the data is official."
+ upload = self._upload_in_flight(data)
+ # Make a fake chunk size smaller than 256 KB.
+ chunk_size = 10
+ assert chunk_size < len(data)
+ upload._chunk_size = chunk_size
+ # Make a fake 308 response.
+ response_headers = {"range": "bytes=0-{:d}".format(chunk_size - 1)}
+ transport = self._chunk_mock(http.client.PERMANENT_REDIRECT, response_headers)
+ # Check the state before the request.
+ assert upload._bytes_uploaded == 0
+
+ # Make request and check the return value (against the mock).
+ response = upload.transmit_next_chunk(transport)
+ assert response is transport.request.return_value
+ # Check that the state has been updated.
+ assert upload._bytes_uploaded == chunk_size
+ # Make sure the mock was called as expected.
+ payload = data[:chunk_size]
+ content_range = "bytes 0-{:d}/{:d}".format(chunk_size - 1, len(data))
+ expected_headers = {
+ "content-range": content_range,
+ "content-type": BASIC_CONTENT,
+ }
+ transport.request.assert_called_once_with(
+ "PUT",
+ upload.resumable_url,
+ data=payload,
+ headers=expected_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ def test_transmit_next_chunk_w_custom_timeout(self):
+ data = b"This time the data is official."
+ upload = self._upload_in_flight(data)
+
+ # Make a fake chunk size smaller than 256 KB.
+ chunk_size = 10
+ upload._chunk_size = chunk_size
+
+ # Make a fake 308 response.
+ response_headers = {"range": "bytes=0-{:d}".format(chunk_size - 1)}
+ transport = self._chunk_mock(http.client.PERMANENT_REDIRECT, response_headers)
+
+ # Make request and check the return value (against the mock).
+ upload.transmit_next_chunk(transport, timeout=12.6)
+
+ # Make sure timeout was passed to the transport
+ payload = data[:chunk_size]
+ content_range = "bytes 0-{:d}/{:d}".format(chunk_size - 1, len(data))
+ expected_headers = {
+ "content-range": content_range,
+ "content-type": BASIC_CONTENT,
+ }
+ transport.request.assert_called_once_with(
+ "PUT",
+ upload.resumable_url,
+ data=payload,
+ headers=expected_headers,
+ timeout=12.6,
+ )
+
+ def test_recover(self):
+ upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ upload._invalid = True # Make sure invalid.
+ upload._stream = mock.Mock(spec=["seek"])
+ upload._resumable_url = "http://test.invalid?upload_id=big-deal"
+
+ end = 55555
+ headers = {"range": "bytes=0-{:d}".format(end)}
+ transport = self._chunk_mock(http.client.PERMANENT_REDIRECT, headers)
+
+ ret_val = upload.recover(transport)
+ assert ret_val is transport.request.return_value
+ # Check the state of ``upload`` after.
+ assert upload.bytes_uploaded == end + 1
+ assert not upload.invalid
+ upload._stream.seek.assert_called_once_with(end + 1)
+ expected_headers = {"content-range": "bytes */*"}
+ transport.request.assert_called_once_with(
+ "PUT",
+ upload.resumable_url,
+ data=None,
+ headers=expected_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+
+def test_mpu_container():
+ container = upload_mod.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename)
+
+ response_text = EXAMPLE_XML_MPU_INITIATE_TEXT_TEMPLATE.format(upload_id=UPLOAD_ID)
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response(text=response_text)
+ container.initiate(transport, BASIC_CONTENT)
+ assert container.upload_id == UPLOAD_ID
+
+ for part, etag in PARTS.items():
+ container.register_part(part, etag)
+
+ assert container._parts == PARTS
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response()
+ container.finalize(transport)
+ assert container.finished
+
+
+def test_mpu_container_cancel():
+ container = upload_mod.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL, filename, upload_id=UPLOAD_ID
+ )
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response(status_code=204)
+ container.cancel(transport)
+
+
+def test_mpu_part(filename):
+ part = upload_mod.XMLMPUPart(EXAMPLE_XML_UPLOAD_URL, UPLOAD_ID, filename, 0, 128, 1)
+
+ transport = mock.Mock(spec=["request"])
+ transport.request.return_value = _make_response(headers={"etag": PARTS[1]})
+
+ part.upload(transport)
+
+ assert part.finished
+ assert part.etag == PARTS[1]
+
+
+def _make_response(status_code=http.client.OK, headers=None, text=None):
+ headers = headers or {}
+ return mock.Mock(
+ headers=headers,
+ status_code=status_code,
+ text=text,
+ spec=["headers", "status_code", "text"],
+ )
diff --git a/packages/google-resumable-media/tests/unit/test__download.py b/packages/google-resumable-media/tests/unit/test__download.py
new file mode 100644
index 000000000000..21a232eb04cd
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/test__download.py
@@ -0,0 +1,754 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+
+from unittest import mock
+import pytest # type: ignore
+
+from google.resumable_media import _download
+from google.resumable_media import common
+
+
+EXAMPLE_URL = (
+ "https://www.googleapis.com/download/storage/v1/b/{BUCKET}/o/{OBJECT}?alt=media"
+)
+
+
+class TestDownloadBase(object):
+ def test_constructor_defaults(self):
+ download = _download.DownloadBase(EXAMPLE_URL)
+ assert download.media_url == EXAMPLE_URL
+ assert download._stream is None
+ assert download.start is None
+ assert download.end is None
+ assert download._headers == {}
+ assert not download._finished
+ _check_retry_strategy(download)
+
+ def test_constructor_explicit(self):
+ start = 11
+ end = 10001
+ headers = {"foof": "barf"}
+ download = _download.DownloadBase(
+ EXAMPLE_URL,
+ stream=mock.sentinel.stream,
+ start=start,
+ end=end,
+ headers=headers,
+ )
+ assert download.media_url == EXAMPLE_URL
+ assert download._stream is mock.sentinel.stream
+ assert download.start == start
+ assert download.end == end
+ assert download._headers is headers
+ assert not download._finished
+ _check_retry_strategy(download)
+
+ def test_finished_property(self):
+ download = _download.DownloadBase(EXAMPLE_URL)
+ # Default value of @property.
+ assert not download.finished
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ download.finished = False
+
+ # Set it privately and then check the @property.
+ download._finished = True
+ assert download.finished
+
+ def test__get_status_code(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _download.DownloadBase._get_status_code(None)
+
+ exc_info.match("virtual")
+
+ def test__get_headers(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _download.DownloadBase._get_headers(None)
+
+ exc_info.match("virtual")
+
+ def test__get_body(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _download.DownloadBase._get_body(None)
+
+ exc_info.match("virtual")
+
+
+class TestDownload(object):
+ def test__prepare_request_already_finished(self):
+ download = _download.Download(EXAMPLE_URL)
+ download._finished = True
+ with pytest.raises(ValueError):
+ download._prepare_request()
+
+ def test__prepare_request(self):
+ download1 = _download.Download(EXAMPLE_URL)
+ method1, url1, payload1, headers1 = download1._prepare_request()
+ assert method1 == "GET"
+ assert url1 == EXAMPLE_URL
+ assert payload1 is None
+ assert headers1 == {}
+
+ download2 = _download.Download(EXAMPLE_URL, start=53)
+ method2, url2, payload2, headers2 = download2._prepare_request()
+ assert method2 == "GET"
+ assert url2 == EXAMPLE_URL
+ assert payload2 is None
+ assert headers2 == {"range": "bytes=53-"}
+
+ def test__prepare_request_with_headers(self):
+ headers = {"spoonge": "borb"}
+ download = _download.Download(EXAMPLE_URL, start=11, end=111, headers=headers)
+ method, url, payload, new_headers = download._prepare_request()
+ assert method == "GET"
+ assert url == EXAMPLE_URL
+ assert payload is None
+ assert new_headers is headers
+ assert headers == {"range": "bytes=11-111", "spoonge": "borb"}
+
+ def test__process_response(self):
+ download = _download.Download(EXAMPLE_URL)
+ _fix_up_virtual(download)
+
+ # Make sure **not finished** before.
+ assert not download.finished
+ response = mock.Mock(status_code=int(http.client.OK), spec=["status_code"])
+ ret_val = download._process_response(response)
+ assert ret_val is None
+ # Make sure **finished** after.
+ assert download.finished
+
+ def test__process_response_bad_status(self):
+ download = _download.Download(EXAMPLE_URL)
+ _fix_up_virtual(download)
+
+ # Make sure **not finished** before.
+ assert not download.finished
+ response = mock.Mock(
+ status_code=int(http.client.NOT_FOUND), spec=["status_code"]
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == response.status_code
+ assert error.args[3] == http.client.OK
+ assert error.args[4] == http.client.PARTIAL_CONTENT
+ # Make sure **finished** even after a failure.
+ assert download.finished
+
+ def test_consume(self):
+ download = _download.Download(EXAMPLE_URL)
+ with pytest.raises(NotImplementedError) as exc_info:
+ download.consume(None)
+
+ exc_info.match("virtual")
+
+
+class TestChunkedDownload(object):
+ def test_constructor_defaults(self):
+ chunk_size = 256
+ stream = mock.sentinel.stream
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ assert download.media_url == EXAMPLE_URL
+ assert download.chunk_size == chunk_size
+ assert download.start == 0
+ assert download.end is None
+ assert download._headers == {}
+ assert not download._finished
+ _check_retry_strategy(download)
+ assert download._stream is stream
+ assert download._bytes_downloaded == 0
+ assert download._total_bytes is None
+ assert not download._invalid
+
+ def test_constructor_bad_start(self):
+ with pytest.raises(ValueError):
+ _download.ChunkedDownload(EXAMPLE_URL, 256, None, start=-11)
+
+ def test_bytes_downloaded_property(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ # Default value of @property.
+ assert download.bytes_downloaded == 0
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ download.bytes_downloaded = 1024
+
+ # Set it privately and then check the @property.
+ download._bytes_downloaded = 128
+ assert download.bytes_downloaded == 128
+
+ def test_total_bytes_property(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ # Default value of @property.
+ assert download.total_bytes is None
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ download.total_bytes = 65536
+
+ # Set it privately and then check the @property.
+ download._total_bytes = 8192
+ assert download.total_bytes == 8192
+
+ def test__get_byte_range(self):
+ chunk_size = 512
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
+ curr_start, curr_end = download._get_byte_range()
+ assert curr_start == 0
+ assert curr_end == chunk_size - 1
+
+ def test__get_byte_range_with_end(self):
+ chunk_size = 512
+ start = 1024
+ end = 1151
+ download = _download.ChunkedDownload(
+ EXAMPLE_URL, chunk_size, None, start=start, end=end
+ )
+ curr_start, curr_end = download._get_byte_range()
+ assert curr_start == start
+ assert curr_end == end
+ # Make sure this is less than the chunk size.
+ actual_size = curr_end - curr_start + 1
+ assert actual_size < chunk_size
+
+ def test__get_byte_range_with_total_bytes(self):
+ chunk_size = 512
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
+ total_bytes = 207
+ download._total_bytes = total_bytes
+ curr_start, curr_end = download._get_byte_range()
+ assert curr_start == 0
+ assert curr_end == total_bytes - 1
+ # Make sure this is less than the chunk size.
+ actual_size = curr_end - curr_start + 1
+ assert actual_size < chunk_size
+
+ @staticmethod
+ def _response_content_range(start_byte, end_byte, total_bytes):
+ return "bytes {:d}-{:d}/{:d}".format(start_byte, end_byte, total_bytes)
+
+ def _response_headers(self, start_byte, end_byte, total_bytes):
+ content_length = end_byte - start_byte + 1
+ resp_range = self._response_content_range(start_byte, end_byte, total_bytes)
+ return {
+ "content-length": "{:d}".format(content_length),
+ "content-range": resp_range,
+ }
+
+ def _mock_response(
+ self, start_byte, end_byte, total_bytes, content=None, status_code=None
+ ):
+ response_headers = self._response_headers(start_byte, end_byte, total_bytes)
+ return mock.Mock(
+ content=content,
+ headers=response_headers,
+ status_code=status_code,
+ spec=["content", "headers", "status_code"],
+ )
+
+ def test__prepare_request_already_finished(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 64, None)
+ download._finished = True
+ with pytest.raises(ValueError) as exc_info:
+ download._prepare_request()
+
+ assert exc_info.match("Download has finished.")
+
+ def test__prepare_request_invalid(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 64, None)
+ download._invalid = True
+ with pytest.raises(ValueError) as exc_info:
+ download._prepare_request()
+
+ assert exc_info.match("Download is invalid and cannot be re-used.")
+
+ def test__prepare_request(self):
+ chunk_size = 2048
+ download1 = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
+ method1, url1, payload1, headers1 = download1._prepare_request()
+ assert method1 == "GET"
+ assert url1 == EXAMPLE_URL
+ assert payload1 is None
+ assert headers1 == {"range": "bytes=0-2047"}
+
+ download2 = _download.ChunkedDownload(
+ EXAMPLE_URL, chunk_size, None, start=19991
+ )
+ download2._total_bytes = 20101
+ method2, url2, payload2, headers2 = download2._prepare_request()
+ assert method2 == "GET"
+ assert url2 == EXAMPLE_URL
+ assert payload2 is None
+ assert headers2 == {"range": "bytes=19991-20100"}
+
+ def test__prepare_request_with_headers(self):
+ chunk_size = 2048
+ headers = {"patrizio": "Starf-ish"}
+ download = _download.ChunkedDownload(
+ EXAMPLE_URL, chunk_size, None, headers=headers
+ )
+ method, url, payload, new_headers = download._prepare_request()
+ assert method == "GET"
+ assert url == EXAMPLE_URL
+ assert payload is None
+ assert new_headers is headers
+ expected = {"patrizio": "Starf-ish", "range": "bytes=0-2047"}
+ assert headers == expected
+
+ def test__make_invalid(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 512, None)
+ assert not download.invalid
+ download._make_invalid()
+ assert download.invalid
+
+ def test__process_response(self):
+ data = b"1234xyztL" * 37 # 9 * 37 == 33
+ chunk_size = len(data)
+ stream = io.BytesIO()
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ already = 22
+ download._bytes_downloaded = already
+ total_bytes = 4444
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == already
+ assert download.total_bytes is None
+ # Actually call the method to update.
+ response = self._mock_response(
+ already,
+ already + chunk_size - 1,
+ total_bytes,
+ content=data,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ )
+ download._process_response(response)
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == already + chunk_size
+ assert download.total_bytes == total_bytes
+ assert stream.getvalue() == data
+
+ def test__process_response_transfer_encoding(self):
+ data = b"1234xyztL" * 37
+ chunk_size = len(data)
+ stream = io.BytesIO()
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ already = 22
+ download._bytes_downloaded = already
+ total_bytes = 4444
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == already
+ assert download.total_bytes is None
+ assert not download.invalid
+ # Actually call the method to update.
+ response = self._mock_response(
+ already,
+ already + chunk_size - 1,
+ total_bytes,
+ content=data,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ )
+ response.headers["transfer-encoding"] = "chunked"
+ del response.headers["content-length"]
+ download._process_response(response)
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == already + chunk_size
+ assert download.total_bytes == total_bytes
+ assert stream.getvalue() == data
+
+ def test__process_response_bad_status(self):
+ chunk_size = 384
+ stream = mock.Mock(spec=["write"])
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ total_bytes = 300
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually call the method to update.
+ response = self._mock_response(
+ 0, total_bytes - 1, total_bytes, status_code=int(http.client.NOT_FOUND)
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == response.status_code
+ assert error.args[3] == http.client.OK
+ assert error.args[4] == http.client.PARTIAL_CONTENT
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert download.invalid
+ stream.write.assert_not_called()
+
+ def test__process_response_missing_content_length(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ _fix_up_virtual(download)
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert not download.invalid
+ # Actually call the method to update.
+ response = mock.Mock(
+ headers={"content-range": "bytes 0-99/99"},
+ status_code=int(http.client.PARTIAL_CONTENT),
+ content=b"DEADBEEF",
+ spec=["headers", "status_code", "content"],
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 2
+ assert error.args[1] == "content-length"
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert download.invalid
+
+ def test__process_response_bad_content_range(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ _fix_up_virtual(download)
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert not download.invalid
+ # Actually call the method to update.
+ data = b"stuff"
+ headers = {
+ "content-length": "{:d}".format(len(data)),
+ "content-range": "kites x-y/58",
+ }
+ response = mock.Mock(
+ content=data,
+ headers=headers,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ spec=["content", "headers", "status_code"],
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == headers["content-range"]
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert download.invalid
+
+ def test__process_response_body_wrong_length(self):
+ chunk_size = 10
+ stream = mock.Mock(spec=["write"])
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ total_bytes = 100
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually call the method to update.
+ data = b"not 10"
+ response = self._mock_response(
+ 0,
+ chunk_size - 1,
+ total_bytes,
+ content=data,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[2] == chunk_size
+ assert error.args[4] == len(data)
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert download.invalid
+ stream.write.assert_not_called()
+
+ def test__process_response_when_finished(self):
+ chunk_size = 256
+ stream = io.BytesIO()
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ total_bytes = 200
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually call the method to update.
+ data = b"abcd" * 50 # 4 * 50 == 200
+ response = self._mock_response(
+ 0,
+ total_bytes - 1,
+ total_bytes,
+ content=data,
+ status_code=int(http.client.OK),
+ )
+ download._process_response(response)
+ # Check internal state after.
+ assert download.finished
+ assert download.bytes_downloaded == total_bytes
+ assert total_bytes < chunk_size
+ assert download.total_bytes == total_bytes
+ assert stream.getvalue() == data
+
+ def test__process_response_when_reaching_end(self):
+ chunk_size = 8192
+ end = 65000
+ stream = io.BytesIO()
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream, end=end)
+ _fix_up_virtual(download)
+
+ download._bytes_downloaded = 7 * chunk_size
+ download._total_bytes = 8 * chunk_size
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 7 * chunk_size
+ assert download.total_bytes == 8 * chunk_size
+ # Actually call the method to update.
+ expected_size = end - 7 * chunk_size + 1
+ data = b"B" * expected_size
+ response = self._mock_response(
+ 7 * chunk_size,
+ end,
+ 8 * chunk_size,
+ content=data,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ )
+ download._process_response(response)
+ # Check internal state after.
+ assert download.finished
+ assert download.bytes_downloaded == end + 1
+ assert download.bytes_downloaded < download.total_bytes
+ assert download.total_bytes == 8 * chunk_size
+ assert stream.getvalue() == data
+
+ def test__process_response_when_content_range_is_zero(self):
+ chunk_size = 10
+ stream = mock.Mock(spec=["write"])
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ content_range = _download._ZERO_CONTENT_RANGE_HEADER
+ headers = {"content-range": content_range}
+ status_code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE
+ response = mock.Mock(
+ headers=headers, status_code=status_code, spec=["headers", "status_code"]
+ )
+ download._process_response(response)
+ stream.write.assert_not_called()
+ assert download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+
+ def test_consume_next_chunk(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ with pytest.raises(NotImplementedError) as exc_info:
+ download.consume_next_chunk(None)
+
+ exc_info.match("virtual")
+
+
+class Test__add_bytes_range(object):
+ def test_do_nothing(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(None, None, headers)
+ assert ret_val is None
+ assert headers == {}
+
+ def test_both_vals(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(17, 1997, headers)
+ assert ret_val is None
+ assert headers == {"range": "bytes=17-1997"}
+
+ def test_end_only(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(None, 909, headers)
+ assert ret_val is None
+ assert headers == {"range": "bytes=0-909"}
+
+ def test_start_only(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(3735928559, None, headers)
+ assert ret_val is None
+ assert headers == {"range": "bytes=3735928559-"}
+
+ def test_start_as_offset(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(-123454321, None, headers)
+ assert ret_val is None
+ assert headers == {"range": "bytes=-123454321"}
+
+
+class Test_get_range_info(object):
+ @staticmethod
+ def _make_response(content_range):
+ headers = {"content-range": content_range}
+ return mock.Mock(headers=headers, spec=["headers"])
+
+ def _success_helper(self, **kwargs):
+ content_range = "Bytes 7-11/42"
+ response = self._make_response(content_range)
+ start_byte, end_byte, total_bytes = _download.get_range_info(
+ response, _get_headers, **kwargs
+ )
+ assert start_byte == 7
+ assert end_byte == 11
+ assert total_bytes == 42
+
+ def test_success(self):
+ self._success_helper()
+
+ def test_success_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._success_helper(callback=callback)
+ callback.assert_not_called()
+
+ def _failure_helper(self, **kwargs):
+ content_range = "nope x-6/y"
+ response = self._make_response(content_range)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _download.get_range_info(response, _get_headers, **kwargs)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == content_range
+
+ def test_failure(self):
+ self._failure_helper()
+
+ def test_failure_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._failure_helper(callback=callback)
+ callback.assert_called_once_with()
+
+ def _missing_header_helper(self, **kwargs):
+ response = mock.Mock(headers={}, spec=["headers"])
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _download.get_range_info(response, _get_headers, **kwargs)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 2
+ assert error.args[1] == "content-range"
+
+ def test_missing_header(self):
+ self._missing_header_helper()
+
+ def test_missing_header_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._missing_header_helper(callback=callback)
+ callback.assert_called_once_with()
+
+
+class Test__check_for_zero_content_range(object):
+ @staticmethod
+ def _make_response(content_range, status_code):
+ headers = {"content-range": content_range}
+ return mock.Mock(
+ headers=headers, status_code=status_code, spec=["headers", "status_code"]
+ )
+
+ def test_status_code_416_and_test_content_range_zero_both(self):
+ content_range = _download._ZERO_CONTENT_RANGE_HEADER
+ status_code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE
+ response = self._make_response(content_range, status_code)
+ assert _download._check_for_zero_content_range(
+ response, _get_status_code, _get_headers
+ )
+
+ def test_status_code_416_only(self):
+ content_range = "bytes 2-5/3"
+ status_code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE
+ response = self._make_response(content_range, status_code)
+ assert not _download._check_for_zero_content_range(
+ response, _get_status_code, _get_headers
+ )
+
+ def test_content_range_zero_only(self):
+ content_range = _download._ZERO_CONTENT_RANGE_HEADER
+ status_code = http.client.OK
+ response = self._make_response(content_range, status_code)
+ assert not _download._check_for_zero_content_range(
+ response, _get_status_code, _get_headers
+ )
+
+
+def _get_status_code(response):
+ return response.status_code
+
+
+def _get_headers(response):
+ return response.headers
+
+
+def _get_body(response):
+ return response.content
+
+
+def _fix_up_virtual(download):
+ download._get_status_code = _get_status_code
+ download._get_headers = _get_headers
+ download._get_body = _get_body
+
+
+def _check_retry_strategy(download):
+ retry_strategy = download._retry_strategy
+ assert isinstance(retry_strategy, common.RetryStrategy)
+ assert retry_strategy.max_sleep == common.MAX_SLEEP
+ assert retry_strategy.max_cumulative_retry == common.MAX_CUMULATIVE_RETRY
+ assert retry_strategy.max_retries is None
diff --git a/packages/google-resumable-media/tests/unit/test__helpers.py b/packages/google-resumable-media/tests/unit/test__helpers.py
new file mode 100644
index 000000000000..98cbc1f99684
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/test__helpers.py
@@ -0,0 +1,506 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+
+import hashlib
+import http.client
+
+from unittest import mock
+import pytest # type: ignore
+
+from google.resumable_media import _helpers
+from google.resumable_media import common
+
+
+def test_do_nothing():
+ ret_val = _helpers.do_nothing()
+ assert ret_val is None
+
+
+class Test_header_required(object):
+ def _success_helper(self, **kwargs):
+ name = "some-header"
+ value = "The Right Hand Side"
+ headers = {name: value, "other-name": "other-value"}
+ response = mock.Mock(headers=headers, spec=["headers"])
+ result = _helpers.header_required(response, name, _get_headers, **kwargs)
+ assert result == value
+
+ def test_success(self):
+ self._success_helper()
+
+ def test_success_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._success_helper(callback=callback)
+ callback.assert_not_called()
+
+ def _failure_helper(self, **kwargs):
+ response = mock.Mock(headers={}, spec=["headers"])
+ name = "any-name"
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _helpers.header_required(response, name, _get_headers, **kwargs)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 2
+ assert error.args[1] == name
+
+ def test_failure(self):
+ self._failure_helper()
+
+ def test_failure_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._failure_helper(callback=callback)
+ callback.assert_called_once_with()
+
+
+class Test_require_status_code(object):
+ @staticmethod
+ def _get_status_code(response):
+ return response.status_code
+
+ def test_success(self):
+ status_codes = (http.client.OK, http.client.CREATED)
+ acceptable = (
+ http.client.OK,
+ int(http.client.OK),
+ http.client.CREATED,
+ int(http.client.CREATED),
+ )
+ for value in acceptable:
+ response = _make_response(value)
+ status_code = _helpers.require_status_code(
+ response, status_codes, self._get_status_code
+ )
+ assert value == status_code
+
+ def test_success_with_callback(self):
+ status_codes = (http.client.OK,)
+ response = _make_response(http.client.OK)
+ callback = mock.Mock(spec=[])
+ status_code = _helpers.require_status_code(
+ response, status_codes, self._get_status_code, callback=callback
+ )
+ assert status_code == http.client.OK
+ callback.assert_not_called()
+
+ def test_failure(self):
+ status_codes = (http.client.CREATED, http.client.NO_CONTENT)
+ response = _make_response(http.client.OK)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _helpers.require_status_code(response, status_codes, self._get_status_code)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == response.status_code
+ assert error.args[3:] == status_codes
+
+ def test_failure_with_callback(self):
+ status_codes = (http.client.OK,)
+ response = _make_response(http.client.NOT_FOUND)
+ callback = mock.Mock(spec=[])
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _helpers.require_status_code(
+ response, status_codes, self._get_status_code, callback=callback
+ )
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 4
+ assert error.args[1] == response.status_code
+ assert error.args[3:] == status_codes
+ callback.assert_called_once_with()
+
+ def test_retryable_failure_without_callback(self):
+ status_codes = (http.client.OK,)
+ retryable_responses = [
+ _make_response(status_code) for status_code in common.RETRYABLE
+ ]
+ callback = mock.Mock(spec=[])
+ for retryable_response in retryable_responses:
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _helpers.require_status_code(
+ retryable_response,
+ status_codes,
+ self._get_status_code,
+ callback=callback,
+ )
+
+ error = exc_info.value
+ assert error.response is retryable_response
+ assert len(error.args) == 4
+ assert error.args[1] == retryable_response.status_code
+ assert error.args[3:] == status_codes
+ callback.assert_not_called()
+
+
+class Test_calculate_retry_wait(object):
+ @mock.patch("random.randint", return_value=125)
+ def test_past_limit(self, randint_mock):
+ base_wait, wait_time = _helpers.calculate_retry_wait(70.0, 64.0)
+
+ assert base_wait == 64.0
+ assert wait_time == 64.125
+ randint_mock.assert_called_once_with(0, 1000)
+
+ @mock.patch("random.randint", return_value=250)
+ def test_at_limit(self, randint_mock):
+ base_wait, wait_time = _helpers.calculate_retry_wait(50.0, 50.0)
+
+ assert base_wait == 50.0
+ assert wait_time == 50.25
+ randint_mock.assert_called_once_with(0, 1000)
+
+ @mock.patch("random.randint", return_value=875)
+ def test_under_limit(self, randint_mock):
+ base_wait, wait_time = _helpers.calculate_retry_wait(16.0, 33.0)
+
+ assert base_wait == 32.0
+ assert wait_time == 32.875
+ randint_mock.assert_called_once_with(0, 1000)
+
+ @mock.patch("random.randint", return_value=875)
+ def test_custom_multiplier(self, randint_mock):
+ base_wait, wait_time = _helpers.calculate_retry_wait(16.0, 64.0, 3)
+
+ assert base_wait == 48.0
+ assert wait_time == 48.875
+ randint_mock.assert_called_once_with(0, 1000)
+
+
+def _make_response(status_code):
+ return mock.Mock(status_code=status_code, spec=["status_code"])
+
+
+def _get_headers(response):
+ return response.headers
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+def test__get_checksum_object(checksum):
+ checksum_object = _helpers._get_checksum_object(checksum)
+
+ checksum_types = {
+ "md5": type(hashlib.md5()),
+ "crc32c": type(_helpers._get_crc32c_object()),
+ None: type(None),
+ }
+ assert isinstance(checksum_object, checksum_types[checksum])
+
+
+def test__get_checksum_object_invalid():
+ with pytest.raises(ValueError):
+ _helpers._get_checksum_object("invalid")
+
+
+@mock.patch("builtins.__import__")
+def test__get_crc32_object_wo_google_crc32c_wo_crcmod(mock_import):
+ mock_import.side_effect = ImportError("testing")
+
+ with pytest.raises(ImportError):
+ _helpers._get_crc32c_object()
+
+ expected_calls = [
+ mock.call("google_crc32c", mock.ANY, None, None, 0),
+ mock.call("crcmod", mock.ANY, None, None, 0),
+ ]
+ mock_import.assert_has_calls(expected_calls)
+
+
+@mock.patch("builtins.__import__")
+def test__get_crc32_object_w_google_crc32c(mock_import):
+ google_crc32c = mock.Mock(spec=["Checksum"])
+ mock_import.return_value = google_crc32c
+
+ found = _helpers._get_crc32c_object()
+
+ assert found is google_crc32c.Checksum.return_value
+ google_crc32c.Checksum.assert_called_once_with()
+
+ mock_import.assert_called_once_with("google_crc32c", mock.ANY, None, None, 0)
+
+
+@mock.patch("builtins.__import__")
+def test__get_crc32_object_wo_google_crc32c_w_crcmod(mock_import):
+ crcmod = mock.Mock(spec=["predefined", "crcmod"])
+ crcmod.predefined = mock.Mock(spec=["Crc"])
+ crcmod.crcmod = mock.Mock(spec=["_usingExtension"])
+ mock_import.side_effect = [ImportError("testing"), crcmod, crcmod.crcmod]
+
+ found = _helpers._get_crc32c_object()
+
+ assert found is crcmod.predefined.Crc.return_value
+ crcmod.predefined.Crc.assert_called_once_with("crc-32c")
+
+ expected_calls = [
+ mock.call("google_crc32c", mock.ANY, None, None, 0),
+ mock.call("crcmod", mock.ANY, None, None, 0),
+ mock.call("crcmod.crcmod", mock.ANY, {}, ["_usingExtension"], 0),
+ ]
+ mock_import.assert_has_calls(expected_calls)
+
+
+@pytest.mark.filterwarnings("ignore::RuntimeWarning")
+@mock.patch("builtins.__import__")
+def test__is_fast_crcmod_wo_extension_warning(mock_import):
+ crcmod = mock.Mock(spec=["crcmod"])
+ crcmod.crcmod = mock.Mock(spec=["_usingExtension"])
+ crcmod.crcmod._usingExtension = False
+ mock_import.return_value = crcmod.crcmod
+
+ assert not _helpers._is_fast_crcmod()
+
+ mock_import.assert_called_once_with(
+ "crcmod.crcmod",
+ mock.ANY,
+ {},
+ ["_usingExtension"],
+ 0,
+ )
+
+
+@mock.patch("builtins.__import__")
+def test__is_fast_crcmod_w_extension(mock_import):
+ crcmod = mock.Mock(spec=["crcmod"])
+ crcmod.crcmod = mock.Mock(spec=["_usingExtension"])
+ crcmod.crcmod._usingExtension = True
+ mock_import.return_value = crcmod.crcmod
+
+ assert _helpers._is_fast_crcmod()
+
+
+def test__DoNothingHash():
+ do_nothing_hash = _helpers._DoNothingHash()
+ return_value = do_nothing_hash.update(b"some data")
+ assert return_value is None
+
+
+class Test__get_expected_checksum(object):
+ @pytest.mark.parametrize("template", ["crc32c={},md5={}", "crc32c={}, md5={}"])
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @mock.patch("google.resumable_media._helpers._LOGGER")
+ def test__w_header_present(self, _LOGGER, template, checksum):
+ checksums = {"md5": "b2twdXNodGhpc2J1dHRvbg==", "crc32c": "3q2+7w=="}
+ header_value = template.format(checksums["crc32c"], checksums["md5"])
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(headers=headers)
+
+ def _get_headers(response):
+ return response.headers
+
+ url = "https://example.com/"
+ expected_checksum, checksum_obj = _helpers._get_expected_checksum(
+ response, _get_headers, url, checksum_type=checksum
+ )
+ assert expected_checksum == checksums[checksum]
+
+ checksum_types = {
+ "md5": type(hashlib.md5()),
+ "crc32c": type(_helpers._get_crc32c_object()),
+ }
+ assert isinstance(checksum_obj, checksum_types[checksum])
+
+ _LOGGER.info.assert_not_called()
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @mock.patch("google.resumable_media._helpers._LOGGER")
+ def test__w_header_missing(self, _LOGGER, checksum):
+ headers = {}
+ response = _mock_response(headers=headers)
+
+ def _get_headers(response):
+ return response.headers
+
+ url = "https://example.com/"
+ expected_checksum, checksum_obj = _helpers._get_expected_checksum(
+ response, _get_headers, url, checksum_type=checksum
+ )
+ assert expected_checksum is None
+ assert isinstance(checksum_obj, _helpers._DoNothingHash)
+ expected_msg = _helpers._MISSING_CHECKSUM.format(
+ url, checksum_type=checksum.upper()
+ )
+ _LOGGER.info.assert_called_once_with(expected_msg)
+
+
+class Test__parse_checksum_header(object):
+ CRC32C_CHECKSUM = "3q2+7w=="
+ MD5_CHECKSUM = "c2l4dGVlbmJ5dGVzbG9uZw=="
+
+ def test_empty_value(self):
+ header_value = None
+ response = None
+ md5_header = _helpers._parse_checksum_header(
+ header_value, response, checksum_label="md5"
+ )
+ assert md5_header is None
+ crc32c_header = _helpers._parse_checksum_header(
+ header_value, response, checksum_label="crc32c"
+ )
+ assert crc32c_header is None
+
+ def test_crc32c_only(self):
+ header_value = "crc32c={}".format(self.CRC32C_CHECKSUM)
+ response = None
+ md5_header = _helpers._parse_checksum_header(
+ header_value, response, checksum_label="md5"
+ )
+ assert md5_header is None
+ crc32c_header = _helpers._parse_checksum_header(
+ header_value, response, checksum_label="crc32c"
+ )
+ assert crc32c_header == self.CRC32C_CHECKSUM
+
+ def test_md5_only(self):
+ header_value = "md5={}".format(self.MD5_CHECKSUM)
+ response = None
+ md5_header = _helpers._parse_checksum_header(
+ header_value, response, checksum_label="md5"
+ )
+ assert md5_header == self.MD5_CHECKSUM
+ crc32c_header = _helpers._parse_checksum_header(
+ header_value, response, checksum_label="crc32c"
+ )
+ assert crc32c_header is None
+
+ def test_both_crc32c_and_md5(self):
+ header_value = "crc32c={},md5={}".format(
+ self.CRC32C_CHECKSUM, self.MD5_CHECKSUM
+ )
+ response = None
+ md5_header = _helpers._parse_checksum_header(
+ header_value, response, checksum_label="md5"
+ )
+ assert md5_header == self.MD5_CHECKSUM
+ crc32c_header = _helpers._parse_checksum_header(
+ header_value, response, checksum_label="crc32c"
+ )
+ assert crc32c_header == self.CRC32C_CHECKSUM
+
+ def test_md5_multiple_matches(self):
+ another_checksum = "eW91IGRpZCBXQVQgbm93Pw=="
+ header_value = "md5={},md5={}".format(self.MD5_CHECKSUM, another_checksum)
+ response = mock.sentinel.response
+
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _helpers._parse_checksum_header(
+ header_value, response, checksum_label="md5"
+ )
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == header_value
+ assert error.args[2] == [self.MD5_CHECKSUM, another_checksum]
+
+
+class Test__parse_generation_header(object):
+ GENERATION_VALUE = 1641590104888641
+
+ def test_empty_value(self):
+ headers = {}
+ response = _mock_response(headers=headers)
+ generation_header = _helpers._parse_generation_header(response, _get_headers)
+ assert generation_header is None
+
+ def test_header_value(self):
+ headers = {_helpers._GENERATION_HEADER: self.GENERATION_VALUE}
+ response = _mock_response(headers=headers)
+ generation_header = _helpers._parse_generation_header(response, _get_headers)
+ assert generation_header == self.GENERATION_VALUE
+
+
+class Test__is_decompressive_transcoding(object):
+ def test_empty_value(self):
+ headers = {}
+ response = _mock_response(headers=headers)
+ assert _helpers._is_decompressive_transcoding(response, _get_headers) is False
+
+ def test_gzip_in_headers(self):
+ headers = {_helpers._STORED_CONTENT_ENCODING_HEADER: "gzip"}
+ response = _mock_response(headers=headers)
+ assert _helpers._is_decompressive_transcoding(response, _get_headers) is True
+
+ def test_gzip_not_in_headers(self):
+ headers = {_helpers._STORED_CONTENT_ENCODING_HEADER: "identity"}
+ response = _mock_response(headers=headers)
+ assert _helpers._is_decompressive_transcoding(response, _get_headers) is False
+
+ def test_gzip_w_content_encoding_in_headers(self):
+ headers = {
+ _helpers._STORED_CONTENT_ENCODING_HEADER: "gzip",
+ _helpers.CONTENT_ENCODING_HEADER: "gzip",
+ }
+ response = _mock_response(headers=headers)
+ assert _helpers._is_decompressive_transcoding(response, _get_headers) is False
+
+
+class Test__get_generation_from_url(object):
+ GENERATION_VALUE = 1641590104888641
+ MEDIA_URL = (
+ "https://storage.googleapis.com/storage/v1/b/my-bucket/o/my-object?alt=media"
+ )
+ MEDIA_URL_W_GENERATION = MEDIA_URL + f"&generation={GENERATION_VALUE}"
+
+ def test_empty_value(self):
+ generation = _helpers._get_generation_from_url(self.MEDIA_URL)
+ assert generation is None
+
+ def test_generation_in_url(self):
+ generation = _helpers._get_generation_from_url(self.MEDIA_URL_W_GENERATION)
+ assert generation == self.GENERATION_VALUE
+
+
+class Test__add_query_parameters(object):
+ def test_w_empty_list(self):
+ query_params = {}
+ MEDIA_URL = "https://storage.googleapis.com/storage/v1/b/my-bucket/o/my-object"
+ new_url = _helpers.add_query_parameters(MEDIA_URL, query_params)
+ assert new_url == MEDIA_URL
+
+ def test_wo_existing_qs(self):
+ query_params = {"one": "One", "two": "Two"}
+ MEDIA_URL = "https://storage.googleapis.com/storage/v1/b/my-bucket/o/my-object"
+ expected = "&".join(
+ ["{}={}".format(name, value) for name, value in query_params.items()]
+ )
+ new_url = _helpers.add_query_parameters(MEDIA_URL, query_params)
+ assert new_url == "{}?{}".format(MEDIA_URL, expected)
+
+ def test_w_existing_qs(self):
+ query_params = {"one": "One", "two": "Two"}
+ MEDIA_URL = "https://storage.googleapis.com/storage/v1/b/my-bucket/o/my-object?alt=media"
+ expected = "&".join(
+ ["{}={}".format(name, value) for name, value in query_params.items()]
+ )
+ new_url = _helpers.add_query_parameters(MEDIA_URL, query_params)
+ assert new_url == "{}&{}".format(MEDIA_URL, expected)
+
+
+def test__get_uploaded_checksum_from_headers_error_handling():
+ response = _mock_response({})
+
+ with pytest.raises(ValueError):
+ _helpers._get_uploaded_checksum_from_headers(response, None, "invalid")
+ assert _helpers._get_uploaded_checksum_from_headers(response, None, None) is None
+
+
+def _mock_response(headers):
+ return mock.Mock(
+ headers=headers,
+ status_code=200,
+ spec=["status_code", "headers"],
+ )
diff --git a/packages/google-resumable-media/tests/unit/test__upload.py b/packages/google-resumable-media/tests/unit/test__upload.py
new file mode 100644
index 000000000000..5e1da37d86aa
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/test__upload.py
@@ -0,0 +1,1533 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+import sys
+import tempfile
+
+from unittest import mock
+import pytest # type: ignore
+
+from google.resumable_media import _helpers
+from google.resumable_media import _upload
+from google.resumable_media import common
+
+
+URL_PREFIX = "https://www.googleapis.com/upload/storage/v1/b/{BUCKET}/o"
+SIMPLE_URL = URL_PREFIX + "?uploadType=media&name={OBJECT}"
+MULTIPART_URL = URL_PREFIX + "?uploadType=multipart"
+RESUMABLE_URL = URL_PREFIX + "?uploadType=resumable"
+ONE_MB = 1024 * 1024
+BASIC_CONTENT = "text/plain"
+JSON_TYPE = "application/json; charset=UTF-8"
+JSON_TYPE_LINE = b"content-type: application/json; charset=UTF-8\r\n"
+EXAMPLE_XML_UPLOAD_URL = "https://test-project.storage.googleapis.com/test-bucket"
+EXAMPLE_HEADERS = {"example-key": "example-content"}
+EXAMPLE_XML_MPU_INITIATE_TEXT_TEMPLATE = """
+
+ travel-maps
+ paris.jpg
+ {upload_id}
+
+"""
+UPLOAD_ID = "VXBsb2FkIElEIGZvciBlbHZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA"
+PARTS = {1: "39a59594290b0f9a30662a56d695b71d", 2: "00000000290b0f9a30662a56d695b71d"}
+FILE_DATA = b"testdata" * 128
+
+
+@pytest.fixture(scope="session")
+def filename():
+ with tempfile.NamedTemporaryFile() as f:
+ f.write(FILE_DATA)
+ f.flush()
+ yield f.name
+
+
+class TestUploadBase(object):
+ def test_constructor_defaults(self):
+ upload = _upload.UploadBase(SIMPLE_URL)
+ assert upload.upload_url == SIMPLE_URL
+ assert upload._headers == {}
+ assert not upload._finished
+ _check_retry_strategy(upload)
+
+ def test_constructor_explicit(self):
+ headers = {"spin": "doctors"}
+ upload = _upload.UploadBase(SIMPLE_URL, headers=headers)
+ assert upload.upload_url == SIMPLE_URL
+ assert upload._headers is headers
+ assert not upload._finished
+ _check_retry_strategy(upload)
+
+ def test_finished_property(self):
+ upload = _upload.UploadBase(SIMPLE_URL)
+ # Default value of @property.
+ assert not upload.finished
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.finished = False
+
+ # Set it privately and then check the @property.
+ upload._finished = True
+ assert upload.finished
+
+ def test__process_response_bad_status(self):
+ upload = _upload.UploadBase(SIMPLE_URL)
+ _fix_up_virtual(upload)
+
+ # Make sure **not finished** before.
+ assert not upload.finished
+ status_code = http.client.SERVICE_UNAVAILABLE
+ response = _make_response(status_code=status_code)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 4
+ assert error.args[1] == status_code
+ assert error.args[3] == http.client.OK
+ # Make sure **finished** after (even in failure).
+ assert upload.finished
+
+ def test__process_response(self):
+ upload = _upload.UploadBase(SIMPLE_URL)
+ _fix_up_virtual(upload)
+
+ # Make sure **not finished** before.
+ assert not upload.finished
+ response = _make_response()
+ ret_val = upload._process_response(response)
+ assert ret_val is None
+ # Make sure **finished** after.
+ assert upload.finished
+
+ def test__get_status_code(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _upload.UploadBase._get_status_code(None)
+
+ exc_info.match("virtual")
+
+ def test__get_headers(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _upload.UploadBase._get_headers(None)
+
+ exc_info.match("virtual")
+
+ def test__get_body(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _upload.UploadBase._get_body(None)
+
+ exc_info.match("virtual")
+
+
+class TestSimpleUpload(object):
+ def test__prepare_request_already_finished(self):
+ upload = _upload.SimpleUpload(SIMPLE_URL)
+ upload._finished = True
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request(b"", None)
+
+ exc_info.match("An upload can only be used once.")
+
+ def test__prepare_request_non_bytes_data(self):
+ upload = _upload.SimpleUpload(SIMPLE_URL)
+ assert not upload.finished
+ with pytest.raises(TypeError) as exc_info:
+ upload._prepare_request("", None)
+
+ exc_info.match("must be bytes")
+
+ def test__prepare_request(self):
+ upload = _upload.SimpleUpload(SIMPLE_URL)
+ content_type = "image/jpeg"
+ data = b"cheetos and eetos"
+ method, url, payload, headers = upload._prepare_request(data, content_type)
+
+ assert method == "POST"
+ assert url == SIMPLE_URL
+ assert payload == data
+ assert headers == {"content-type": content_type}
+
+ def test__prepare_request_with_headers(self):
+ headers = {"x-goog-cheetos": "spicy"}
+ upload = _upload.SimpleUpload(SIMPLE_URL, headers=headers)
+ content_type = "image/jpeg"
+ data = b"some stuff"
+ method, url, payload, new_headers = upload._prepare_request(data, content_type)
+
+ assert method == "POST"
+ assert url == SIMPLE_URL
+ assert payload == data
+ assert new_headers is headers
+ expected = {"content-type": content_type, "x-goog-cheetos": "spicy"}
+ assert headers == expected
+
+ def test_transmit(self):
+ upload = _upload.SimpleUpload(SIMPLE_URL)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.transmit(None, None, None)
+
+ exc_info.match("virtual")
+
+
+class TestMultipartUpload(object):
+ def test_constructor_defaults(self):
+ upload = _upload.MultipartUpload(MULTIPART_URL)
+ assert upload.upload_url == MULTIPART_URL
+ assert upload._headers == {}
+ assert upload._checksum_type is None
+ assert not upload._finished
+ _check_retry_strategy(upload)
+
+ def test_constructor_explicit(self):
+ headers = {"spin": "doctors"}
+ upload = _upload.MultipartUpload(MULTIPART_URL, headers=headers, checksum="md5")
+ assert upload.upload_url == MULTIPART_URL
+ assert upload._headers is headers
+ assert upload._checksum_type == "md5"
+ assert not upload._finished
+ _check_retry_strategy(upload)
+
+ def test__prepare_request_already_finished(self):
+ upload = _upload.MultipartUpload(MULTIPART_URL)
+ upload._finished = True
+ with pytest.raises(ValueError):
+ upload._prepare_request(b"Hi", {}, BASIC_CONTENT)
+
+ def test__prepare_request_non_bytes_data(self):
+ data = "Nope not bytes."
+ upload = _upload.MultipartUpload(MULTIPART_URL)
+ with pytest.raises(TypeError):
+ upload._prepare_request(data, {}, BASIC_CONTENT)
+
+ @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==3==")
+ def _prepare_request_helper(
+ self,
+ mock_get_boundary,
+ headers=None,
+ checksum=None,
+ expected_checksum=None,
+ test_overwrite=False,
+ ):
+ upload = _upload.MultipartUpload(
+ MULTIPART_URL, headers=headers, checksum=checksum
+ )
+ data = b"Hi"
+ if test_overwrite and checksum:
+ # Deliberately set metadata that conflicts with the chosen checksum.
+ # This should be fully overwritten by the calculated checksum, so
+ # the output should not change even if this is set.
+ if checksum == "md5":
+ metadata = {"md5Hash": "ZZZZZZZZZZZZZZZZZZZZZZ=="}
+ else:
+ metadata = {"crc32c": "ZZZZZZ=="}
+ else:
+ # To simplify parsing the response, omit other test metadata if a
+ # checksum is specified.
+ metadata = {"Some": "Stuff"} if not checksum else {}
+ content_type = BASIC_CONTENT
+ method, url, payload, new_headers = upload._prepare_request(
+ data, metadata, content_type
+ )
+
+ assert method == "POST"
+ assert url == MULTIPART_URL
+
+ preamble = b"--==3==\r\n" + JSON_TYPE_LINE + b"\r\n"
+
+ if checksum == "md5" and expected_checksum:
+ metadata_payload = '{{"md5Hash": "{}"}}\r\n'.format(
+ expected_checksum
+ ).encode("utf8")
+ elif checksum == "crc32c" and expected_checksum:
+ metadata_payload = '{{"crc32c": "{}"}}\r\n'.format(
+ expected_checksum
+ ).encode("utf8")
+ else:
+ metadata_payload = b'{"Some": "Stuff"}\r\n'
+ remainder = b"--==3==\r\ncontent-type: text/plain\r\n\r\nHi\r\n--==3==--"
+ expected_payload = preamble + metadata_payload + remainder
+
+ assert payload == expected_payload
+ multipart_type = b'multipart/related; boundary="==3=="'
+ mock_get_boundary.assert_called_once_with()
+
+ return new_headers, multipart_type
+
+ def test__prepare_request(self):
+ headers, multipart_type = self._prepare_request_helper()
+ assert headers == {"content-type": multipart_type}
+
+ def test__prepare_request_with_headers(self):
+ headers = {"best": "shirt", "worst": "hat"}
+ new_headers, multipart_type = self._prepare_request_helper(headers=headers)
+ assert new_headers is headers
+ expected_headers = {
+ "best": "shirt",
+ "content-type": multipart_type,
+ "worst": "hat",
+ }
+ assert expected_headers == headers
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__prepare_request_with_checksum(self, checksum):
+ checksums = {
+ "md5": "waUpj5Oeh+j5YqXt/CBpGA==",
+ "crc32c": "ihY6wA==",
+ }
+ headers, multipart_type = self._prepare_request_helper(
+ checksum=checksum, expected_checksum=checksums[checksum]
+ )
+ assert headers == {
+ "content-type": multipart_type,
+ }
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__prepare_request_with_checksum_overwrite(self, checksum):
+ checksums = {
+ "md5": "waUpj5Oeh+j5YqXt/CBpGA==",
+ "crc32c": "ihY6wA==",
+ }
+ headers, multipart_type = self._prepare_request_helper(
+ checksum=checksum,
+ expected_checksum=checksums[checksum],
+ test_overwrite=True,
+ )
+ assert headers == {
+ "content-type": multipart_type,
+ }
+
+ def test_transmit(self):
+ upload = _upload.MultipartUpload(MULTIPART_URL)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.transmit(None, None, None, None)
+
+ exc_info.match("virtual")
+
+
+class TestResumableUpload(object):
+ def test_constructor(self):
+ chunk_size = ONE_MB
+ upload = _upload.ResumableUpload(RESUMABLE_URL, chunk_size)
+ assert upload.upload_url == RESUMABLE_URL
+ assert upload._headers == {}
+ assert not upload._finished
+ _check_retry_strategy(upload)
+ assert upload._chunk_size == chunk_size
+ assert upload._stream is None
+ assert upload._content_type is None
+ assert upload._bytes_uploaded == 0
+ assert upload._bytes_checksummed == 0
+ assert upload._checksum_object is None
+ assert upload._total_bytes is None
+ assert upload._resumable_url is None
+ assert upload._checksum_type is None
+
+ def test_constructor_bad_chunk_size(self):
+ with pytest.raises(ValueError):
+ _upload.ResumableUpload(RESUMABLE_URL, 1)
+
+ def test_invalid_property(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ # Default value of @property.
+ assert not upload.invalid
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.invalid = False
+
+ # Set it privately and then check the @property.
+ upload._invalid = True
+ assert upload.invalid
+
+ def test_chunk_size_property(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ # Default value of @property.
+ assert upload.chunk_size == ONE_MB
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.chunk_size = 17
+
+ # Set it privately and then check the @property.
+ new_size = 102
+ upload._chunk_size = new_size
+ assert upload.chunk_size == new_size
+
+ def test_resumable_url_property(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ # Default value of @property.
+ assert upload.resumable_url is None
+
+ # Make sure we cannot set it on public @property.
+ new_url = "http://test.invalid?upload_id=not-none"
+ with pytest.raises(AttributeError):
+ upload.resumable_url = new_url
+
+ # Set it privately and then check the @property.
+ upload._resumable_url = new_url
+ assert upload.resumable_url == new_url
+
+ def test_bytes_uploaded_property(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ # Default value of @property.
+ assert upload.bytes_uploaded == 0
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.bytes_uploaded = 1024
+
+ # Set it privately and then check the @property.
+ upload._bytes_uploaded = 128
+ assert upload.bytes_uploaded == 128
+
+ def test_total_bytes_property(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ # Default value of @property.
+ assert upload.total_bytes is None
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.total_bytes = 65536
+
+ # Set it privately and then check the @property.
+ upload._total_bytes = 8192
+ assert upload.total_bytes == 8192
+
+ def _prepare_initiate_request_helper(
+ self, upload_url=RESUMABLE_URL, upload_headers=None, **method_kwargs
+ ):
+ data = b"some really big big data."
+ stream = io.BytesIO(data)
+ metadata = {"name": "big-data-file.txt"}
+
+ upload = _upload.ResumableUpload(upload_url, ONE_MB, headers=upload_headers)
+ orig_headers = upload._headers.copy()
+ # Check ``upload``-s state before.
+ assert upload._stream is None
+ assert upload._content_type is None
+ assert upload._total_bytes is None
+ # Call the method and check the output.
+ method, url, payload, headers = upload._prepare_initiate_request(
+ stream, metadata, BASIC_CONTENT, **method_kwargs
+ )
+ assert payload == b'{"name": "big-data-file.txt"}'
+ # Make sure the ``upload``-s state was updated.
+ assert upload._stream == stream
+ assert upload._content_type == BASIC_CONTENT
+ if method_kwargs == {"stream_final": False}:
+ assert upload._total_bytes is None
+ else:
+ assert upload._total_bytes == len(data)
+ # Make sure headers are untouched.
+ assert headers is not upload._headers
+ assert upload._headers == orig_headers
+ assert method == "POST"
+ assert url == upload.upload_url
+ # Make sure the stream is still at the beginning.
+ assert stream.tell() == 0
+
+ return data, headers
+
+ def test__prepare_initiate_request(self):
+ data, headers = self._prepare_initiate_request_helper()
+ expected_headers = {
+ "content-type": JSON_TYPE,
+ "x-upload-content-length": "{:d}".format(len(data)),
+ "x-upload-content-type": BASIC_CONTENT,
+ }
+ assert headers == expected_headers
+
+ def test_prepare_initiate_request_with_signed_url(self):
+ signed_urls = [
+ "https://storage.googleapis.com/b/o?x-goog-signature=123abc",
+ "https://storage.googleapis.com/b/o?X-Goog-Signature=123abc",
+ ]
+ for signed_url in signed_urls:
+ data, headers = self._prepare_initiate_request_helper(
+ upload_url=signed_url,
+ )
+ expected_headers = {
+ "content-type": BASIC_CONTENT,
+ "x-upload-content-length": "{:d}".format(len(data)),
+ }
+ assert headers == expected_headers
+
+ def test__prepare_initiate_request_with_headers(self):
+ # content-type header should be overwritten, the rest should stay
+ headers = {
+ "caviar": "beluga",
+ "top": "quark",
+ "content-type": "application/xhtml",
+ }
+ data, new_headers = self._prepare_initiate_request_helper(
+ upload_headers=headers
+ )
+ expected_headers = {
+ "caviar": "beluga",
+ "content-type": JSON_TYPE,
+ "top": "quark",
+ "x-upload-content-length": "{:d}".format(len(data)),
+ "x-upload-content-type": BASIC_CONTENT,
+ }
+ assert new_headers == expected_headers
+
+ def test__prepare_initiate_request_known_size(self):
+ total_bytes = 25
+ data, headers = self._prepare_initiate_request_helper(total_bytes=total_bytes)
+ assert len(data) == total_bytes
+ expected_headers = {
+ "content-type": "application/json; charset=UTF-8",
+ "x-upload-content-length": "{:d}".format(total_bytes),
+ "x-upload-content-type": BASIC_CONTENT,
+ }
+ assert headers == expected_headers
+
+ def test__prepare_initiate_request_unknown_size(self):
+ _, headers = self._prepare_initiate_request_helper(stream_final=False)
+ expected_headers = {
+ "content-type": "application/json; charset=UTF-8",
+ "x-upload-content-type": BASIC_CONTENT,
+ }
+ assert headers == expected_headers
+
+ def test__prepare_initiate_request_already_initiated(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ # Fake that the upload has been started.
+ upload._resumable_url = "http://test.invalid?upload_id=definitely-started"
+
+ with pytest.raises(ValueError):
+ upload._prepare_initiate_request(io.BytesIO(), {}, BASIC_CONTENT)
+
+ def test__prepare_initiate_request_bad_stream_position(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+
+ stream = io.BytesIO(b"data")
+ stream.seek(1)
+ with pytest.raises(ValueError):
+ upload._prepare_initiate_request(stream, {}, BASIC_CONTENT)
+
+ # Also test a bad object (i.e. non-stream)
+ with pytest.raises(AttributeError):
+ upload._prepare_initiate_request(None, {}, BASIC_CONTENT)
+
+ def test__process_initiate_response_non_200(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ response = _make_response(403)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_initiate_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == 403
+ assert error.args[3] == 200
+ assert error.args[4] == 201
+
+ def test__process_initiate_response(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ headers = {"location": "http://test.invalid?upload_id=kmfeij3234"}
+ response = _make_response(headers=headers)
+ # Check resumable_url before.
+ assert upload._resumable_url is None
+ # Process the actual headers.
+ ret_val = upload._process_initiate_response(response)
+ assert ret_val is None
+ # Check resumable_url after.
+ assert upload._resumable_url == headers["location"]
+
+ def test_initiate(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.initiate(None, None, {}, BASIC_CONTENT)
+
+ exc_info.match("virtual")
+
+ def test__prepare_request_already_finished(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ assert not upload.invalid
+ upload._finished = True
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request()
+
+ assert exc_info.value.args == ("Upload has finished.",)
+
+ def test__prepare_request_invalid(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ assert not upload.finished
+ upload._invalid = True
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request()
+
+ assert exc_info.match("invalid state")
+ assert exc_info.match("recover()")
+
+ def test__prepare_request_not_initiated(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ assert not upload.finished
+ assert not upload.invalid
+ assert upload._resumable_url is None
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request()
+
+ assert exc_info.match("upload has not been initiated")
+ assert exc_info.match("initiate()")
+
+ def test__prepare_request_invalid_stream_state(self):
+ stream = io.BytesIO(b"some data here")
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ upload._stream = stream
+ upload._resumable_url = "http://test.invalid?upload_id=not-none"
+ # Make stream.tell() disagree with bytes_uploaded.
+ upload._bytes_uploaded = 5
+ assert upload.bytes_uploaded != stream.tell()
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request()
+
+ assert exc_info.match("Bytes stream is in unexpected state.")
+
+ @staticmethod
+ def _upload_in_flight(data, headers=None, checksum=None):
+ upload = _upload.ResumableUpload(
+ RESUMABLE_URL, ONE_MB, headers=headers, checksum=checksum
+ )
+ upload._stream = io.BytesIO(data)
+ upload._content_type = BASIC_CONTENT
+ upload._total_bytes = len(data)
+ upload._resumable_url = "http://test.invalid?upload_id=not-none"
+ return upload
+
+ def _prepare_request_helper(self, headers=None, checksum=None):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, headers=headers, checksum=checksum)
+ method, url, payload, new_headers = upload._prepare_request()
+ # Check the response values.
+ assert method == "PUT"
+ assert url == upload.resumable_url
+ assert payload == data
+ # Make sure headers are **NOT** updated
+ assert upload._headers != new_headers
+
+ return new_headers
+
+ def test__prepare_request_success(self):
+ headers = self._prepare_request_helper()
+ expected_headers = {
+ "content-range": "bytes 0-32/33",
+ "content-type": BASIC_CONTENT,
+ }
+ assert headers == expected_headers
+
+ def test__prepare_request_success_with_headers(self):
+ headers = {"keep": "this"}
+ new_headers = self._prepare_request_helper(headers)
+ assert new_headers is not headers
+ expected_headers = {
+ "keep": "this",
+ "content-range": "bytes 0-32/33",
+ "content-type": BASIC_CONTENT,
+ }
+ assert new_headers == expected_headers
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__prepare_request_with_checksum(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ upload._prepare_request()
+ assert upload._checksum_object is not None
+
+ checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ checksum_digest = _helpers.prepare_checksum_digest(
+ upload._checksum_object.digest()
+ )
+ assert checksum_digest == checksums[checksum]
+ assert upload._bytes_checksummed == len(data)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__update_checksum(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 8
+
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 16
+
+ # Continue to the end.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ checksum_digest = _helpers.prepare_checksum_digest(
+ upload._checksum_object.digest()
+ )
+ assert checksum_digest == checksums[checksum]
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__update_checksum_rewind(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 8
+ checksum_checkpoint = upload._checksum_object.digest()
+
+ # Rewind to the beginning.
+ upload._stream.seek(0)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 8
+ assert upload._checksum_object.digest() == checksum_checkpoint
+
+ # Rewind but not to the beginning.
+ upload._stream.seek(4)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 12
+
+ # Continue to the end.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ checksum_digest = _helpers.prepare_checksum_digest(
+ upload._checksum_object.digest()
+ )
+ assert checksum_digest == checksums[checksum]
+
+ def test__update_checksum_none(self):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=None)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._checksum_object is None
+
+ def test__update_checksum_invalid(self):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum="invalid")
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ with pytest.raises(ValueError):
+ upload._update_checksum(start_byte, payload)
+
+ def test__make_invalid(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ assert not upload.invalid
+ upload._make_invalid()
+ assert upload.invalid
+
+ def test__process_resumable_response_bad_status(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ # Make sure the upload is valid before the failure.
+ assert not upload.invalid
+ response = _make_response(status_code=http.client.NOT_FOUND)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_resumable_response(response, None)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == response.status_code
+ assert error.args[3] == http.client.OK
+ assert error.args[4] == http.client.PERMANENT_REDIRECT
+ # Make sure the upload is invalid after the failure.
+ assert upload.invalid
+
+ def test__process_resumable_response_success(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ # Check / set status before.
+ assert upload._bytes_uploaded == 0
+ upload._bytes_uploaded = 20
+ assert not upload._finished
+
+ # Set the response body.
+ bytes_sent = 158
+ total_bytes = upload._bytes_uploaded + bytes_sent
+ response_body = '{{"size": "{:d}"}}'.format(total_bytes)
+ response_body = response_body.encode("utf-8")
+ response = mock.Mock(
+ content=response_body,
+ status_code=http.client.OK,
+ spec=["content", "status_code"],
+ )
+ ret_val = upload._process_resumable_response(response, bytes_sent)
+ assert ret_val is None
+ # Check status after.
+ assert upload._bytes_uploaded == total_bytes
+ assert upload._finished
+
+ def test__process_resumable_response_partial_no_range(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ response = _make_response(status_code=http.client.PERMANENT_REDIRECT)
+ # Make sure the upload is valid before the failure.
+ assert not upload.invalid
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_resumable_response(response, None)
+ # Make sure the upload is invalid after the failure.
+ assert upload.invalid
+
+ # Check the error response.
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 2
+ assert error.args[1] == "range"
+
+ def test__process_resumable_response_partial_bad_range(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ # Make sure the upload is valid before the failure.
+ assert not upload.invalid
+ headers = {"range": "nights 1-81"}
+ response = _make_response(
+ status_code=http.client.PERMANENT_REDIRECT, headers=headers
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_resumable_response(response, 81)
+
+ # Check the error response.
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == headers["range"]
+ # Make sure the upload is invalid after the failure.
+ assert upload.invalid
+
+ def test__process_resumable_response_partial(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ # Check status before.
+ assert upload._bytes_uploaded == 0
+ headers = {"range": "bytes=0-171"}
+ response = _make_response(
+ status_code=http.client.PERMANENT_REDIRECT, headers=headers
+ )
+ ret_val = upload._process_resumable_response(response, 172)
+ assert ret_val is None
+ # Check status after.
+ assert upload._bytes_uploaded == 172
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__validate_checksum_success(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ _fix_up_virtual(upload)
+ # Go ahead and process the entire data in one go for this test.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ # This is only used by _validate_checksum for fetching metadata and
+ # logging.
+ metadata = {"md5Hash": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ response = _make_response(metadata=metadata)
+ upload._finished = True
+
+ assert upload._checksum_object is not None
+ # Test passes if it does not raise an error (no assert needed)
+ upload._validate_checksum(response)
+
+ def test__validate_checksum_none(self):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(b"test", checksum=None)
+ _fix_up_virtual(upload)
+ # Go ahead and process the entire data in one go for this test.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+
+ # This is only used by _validate_checksum for fetching metadata and
+ # logging.
+ metadata = {"md5Hash": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ response = _make_response(metadata=metadata)
+ upload._finished = True
+
+ assert upload._checksum_object is None
+ assert upload._bytes_checksummed == 0
+ # Test passes if it does not raise an error (no assert needed)
+ upload._validate_checksum(response)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__validate_checksum_header_no_match(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ _fix_up_virtual(upload)
+ # Go ahead and process the entire data in one go for this test.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ # For this test, each checksum option will be provided with a valid but
+ # mismatching remote checksum type.
+ if checksum == "crc32c":
+ metadata = {"md5Hash": "GRvfKbqr5klAOwLkxgIf8w=="}
+ else:
+ metadata = {"crc32c": "Qg8thA=="}
+ # This is only used by _validate_checksum for fetching headers and
+ # logging, so it doesn't need to be fleshed out with a response body.
+ response = _make_response(metadata=metadata)
+ upload._finished = True
+
+ assert upload._checksum_object is not None
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._validate_checksum(response)
+
+ error = exc_info.value
+ assert error.response is response
+ message = error.args[0]
+ metadata_key = _helpers._get_metadata_key(checksum)
+ assert (
+ message
+ == _upload._UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(
+ metadata_key
+ )
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__validate_checksum_mismatch(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ _fix_up_virtual(upload)
+ # Go ahead and process the entire data in one go for this test.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ metadata = {
+ "md5Hash": "ZZZZZZZZZZZZZZZZZZZZZZ==",
+ "crc32c": "ZZZZZZ==",
+ }
+ # This is only used by _validate_checksum for fetching headers and
+ # logging, so it doesn't need to be fleshed out with a response body.
+ response = _make_response(metadata=metadata)
+ upload._finished = True
+
+ assert upload._checksum_object is not None
+ # Test passes if it does not raise an error (no assert needed)
+ with pytest.raises(common.DataCorruption) as exc_info:
+ upload._validate_checksum(response)
+
+ error = exc_info.value
+ assert error.response is response
+ message = error.args[0]
+ correct_checksums = {"crc32c": "Qg8thA==", "md5": "GRvfKbqr5klAOwLkxgIf8w=="}
+ metadata_key = _helpers._get_metadata_key(checksum)
+ assert message == _upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
+ checksum.upper(), correct_checksums[checksum], metadata[metadata_key]
+ )
+
+ def test_transmit_next_chunk(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.transmit_next_chunk(None)
+
+ exc_info.match("virtual")
+
+ def test__prepare_recover_request_not_invalid(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ assert not upload.invalid
+
+ method, url, payload, headers = upload._prepare_recover_request()
+ assert method == "PUT"
+ assert url == upload.resumable_url
+ assert payload is None
+ assert headers == {"content-range": "bytes */*"}
+ # Make sure headers are untouched.
+ assert upload._headers == {}
+
+ def test__prepare_recover_request(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ upload._invalid = True
+
+ method, url, payload, headers = upload._prepare_recover_request()
+ assert method == "PUT"
+ assert url == upload.resumable_url
+ assert payload is None
+ assert headers == {"content-range": "bytes */*"}
+ # Make sure headers are untouched.
+ assert upload._headers == {}
+
+ def test__prepare_recover_request_with_headers(self):
+ headers = {"lake": "ocean"}
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB, headers=headers)
+ upload._invalid = True
+
+ method, url, payload, new_headers = upload._prepare_recover_request()
+ assert method == "PUT"
+ assert url == upload.resumable_url
+ assert payload is None
+ assert new_headers == {"content-range": "bytes */*"}
+ # Make sure the ``_headers`` are not incorporated.
+ assert "lake" not in new_headers
+ # Make sure headers are untouched.
+ assert upload._headers == {"lake": "ocean"}
+
+ def test__process_recover_response_bad_status(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ upload._invalid = True
+
+ response = _make_response(status_code=http.client.BAD_REQUEST)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_recover_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 4
+ assert error.args[1] == response.status_code
+ assert error.args[3] == http.client.PERMANENT_REDIRECT
+ # Make sure still invalid.
+ assert upload.invalid
+
+ def test__process_recover_response_no_range(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ upload._invalid = True
+ upload._stream = mock.Mock(spec=["seek"])
+ upload._bytes_uploaded = mock.sentinel.not_zero
+ assert upload.bytes_uploaded != 0
+
+ response = _make_response(status_code=http.client.PERMANENT_REDIRECT)
+ ret_val = upload._process_recover_response(response)
+ assert ret_val is None
+ # Check the state of ``upload`` after.
+ assert upload.bytes_uploaded == 0
+ assert not upload.invalid
+ upload._stream.seek.assert_called_once_with(0)
+
+ def test__process_recover_response_bad_range(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ upload._invalid = True
+ upload._stream = mock.Mock(spec=["seek"])
+ upload._bytes_uploaded = mock.sentinel.not_zero
+
+ headers = {"range": "bites=9-11"}
+ response = _make_response(
+ status_code=http.client.PERMANENT_REDIRECT, headers=headers
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_recover_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == headers["range"]
+ # Check the state of ``upload`` after (untouched).
+ assert upload.bytes_uploaded is mock.sentinel.not_zero
+ assert upload.invalid
+ upload._stream.seek.assert_not_called()
+
+ def test__process_recover_response_with_range(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ _fix_up_virtual(upload)
+
+ upload._invalid = True
+ upload._stream = mock.Mock(spec=["seek"])
+ upload._bytes_uploaded = mock.sentinel.not_zero
+ assert upload.bytes_uploaded != 0
+
+ end = 11
+ headers = {"range": "bytes=0-{:d}".format(end)}
+ response = _make_response(
+ status_code=http.client.PERMANENT_REDIRECT, headers=headers
+ )
+ ret_val = upload._process_recover_response(response)
+ assert ret_val is None
+ # Check the state of ``upload`` after.
+ assert upload.bytes_uploaded == end + 1
+ assert not upload.invalid
+ upload._stream.seek.assert_called_once_with(end + 1)
+
+ def test_recover(self):
+ upload = _upload.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.recover(None)
+
+ exc_info.match("virtual")
+
+
+@mock.patch("random.randrange", return_value=1234567890123456789)
+def test_get_boundary(mock_rand):
+ result = _upload.get_boundary()
+ assert result == b"===============1234567890123456789=="
+ mock_rand.assert_called_once_with(sys.maxsize)
+
+
+class Test_construct_multipart_request(object):
+ @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==1==")
+ def test_binary(self, mock_get_boundary):
+ data = b"By nary day tuh"
+ metadata = {"name": "hi-file.bin"}
+ content_type = "application/octet-stream"
+ payload, multipart_boundary = _upload.construct_multipart_request(
+ data, metadata, content_type
+ )
+
+ assert multipart_boundary == mock_get_boundary.return_value
+ expected_payload = (
+ b"--==1==\r\n" + JSON_TYPE_LINE + b"\r\n"
+ b'{"name": "hi-file.bin"}\r\n'
+ b"--==1==\r\n"
+ b"content-type: application/octet-stream\r\n"
+ b"\r\n"
+ b"By nary day tuh\r\n"
+ b"--==1==--"
+ )
+ assert payload == expected_payload
+ mock_get_boundary.assert_called_once_with()
+
+ @mock.patch("google.resumable_media._upload.get_boundary", return_value=b"==2==")
+ def test_unicode(self, mock_get_boundary):
+ data_unicode = "\N{SNOWMAN}"
+ # construct_multipart_request( ASSUMES callers pass bytes.
+ data = data_unicode.encode("utf-8")
+ metadata = {"name": "snowman.txt"}
+ content_type = BASIC_CONTENT
+ payload, multipart_boundary = _upload.construct_multipart_request(
+ data, metadata, content_type
+ )
+
+ assert multipart_boundary == mock_get_boundary.return_value
+ expected_payload = (
+ b"--==2==\r\n" + JSON_TYPE_LINE + b"\r\n"
+ b'{"name": "snowman.txt"}\r\n'
+ b"--==2==\r\n"
+ b"content-type: text/plain\r\n"
+ b"\r\n"
+ b"\xe2\x98\x83\r\n"
+ b"--==2==--"
+ )
+ assert payload == expected_payload
+ mock_get_boundary.assert_called_once_with()
+
+
+def test_get_total_bytes():
+ data = b"some data"
+ stream = io.BytesIO(data)
+ # Check position before function call.
+ assert stream.tell() == 0
+ assert _upload.get_total_bytes(stream) == len(data)
+ # Check position after function call.
+ assert stream.tell() == 0
+
+ # Make sure this works just as well when not at beginning.
+ curr_pos = 3
+ stream.seek(curr_pos)
+ assert _upload.get_total_bytes(stream) == len(data)
+ # Check position after function call.
+ assert stream.tell() == curr_pos
+
+
+class Test_get_next_chunk(object):
+ def test_exhausted_known_size(self):
+ data = b"the end"
+ stream = io.BytesIO(data)
+ stream.seek(len(data))
+ with pytest.raises(ValueError) as exc_info:
+ _upload.get_next_chunk(stream, 1, len(data))
+
+ exc_info.match("Stream is already exhausted. There is no content remaining.")
+
+ def test_exhausted_known_size_zero(self):
+ stream = io.BytesIO(b"")
+ answer = _upload.get_next_chunk(stream, 1, 0)
+ assert answer == (0, b"", "bytes */0")
+
+ def test_exhausted_known_size_zero_nonempty(self):
+ stream = io.BytesIO(b"not empty WAT!")
+ with pytest.raises(ValueError) as exc_info:
+ _upload.get_next_chunk(stream, 1, 0)
+ exc_info.match("Stream specified as empty, but produced non-empty content.")
+
+ def test_success_known_size_lt_stream_size(self):
+ data = b"0123456789"
+ stream = io.BytesIO(data)
+ chunk_size = 3
+ total_bytes = len(data) - 2
+
+ # Splits into 3 chunks: 012, 345, 67
+ result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+
+ assert result0 == (0, b"012", "bytes 0-2/8")
+ assert result1 == (3, b"345", "bytes 3-5/8")
+ assert result2 == (6, b"67", "bytes 6-7/8")
+
+ def test_success_known_size(self):
+ data = b"0123456789"
+ stream = io.BytesIO(data)
+ total_bytes = len(data)
+ chunk_size = 3
+ # Splits into 4 chunks: 012, 345, 678, 9
+ result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result3 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ assert result0 == (0, b"012", "bytes 0-2/10")
+ assert result1 == (3, b"345", "bytes 3-5/10")
+ assert result2 == (6, b"678", "bytes 6-8/10")
+ assert result3 == (9, b"9", "bytes 9-9/10")
+ assert stream.tell() == total_bytes
+
+ def test_success_unknown_size(self):
+ data = b"abcdefghij"
+ stream = io.BytesIO(data)
+ chunk_size = 6
+ # Splits into 4 chunks: abcdef, ghij
+ result0 = _upload.get_next_chunk(stream, chunk_size, None)
+ result1 = _upload.get_next_chunk(stream, chunk_size, None)
+ assert result0 == (0, b"abcdef", "bytes 0-5/*")
+ assert result1 == (chunk_size, b"ghij", "bytes 6-9/10")
+ assert stream.tell() == len(data)
+
+ # Do the same when the chunk size evenly divides len(data)
+ stream.seek(0)
+ chunk_size = len(data)
+ # Splits into 2 chunks: `data` and empty string
+ result0 = _upload.get_next_chunk(stream, chunk_size, None)
+ result1 = _upload.get_next_chunk(stream, chunk_size, None)
+ assert result0 == (0, data, "bytes 0-9/*")
+ assert result1 == (len(data), b"", "bytes */10")
+ assert stream.tell() == len(data)
+
+
+class Test_get_content_range(object):
+ def test_known_size(self):
+ result = _upload.get_content_range(5, 10, 40)
+ assert result == "bytes 5-10/40"
+
+ def test_unknown_size(self):
+ result = _upload.get_content_range(1000, 10000, None)
+ assert result == "bytes 1000-10000/*"
+
+
+def test_xml_mpu_container_constructor_and_properties(filename):
+ container = _upload.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename)
+ assert container.upload_url == EXAMPLE_XML_UPLOAD_URL
+ assert container.upload_id is None
+ assert container._headers == {}
+ assert container._parts == {}
+ assert container._filename == filename
+
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL,
+ filename,
+ headers=EXAMPLE_HEADERS,
+ upload_id=UPLOAD_ID,
+ )
+ container._parts = PARTS
+ assert container.upload_url == EXAMPLE_XML_UPLOAD_URL
+ assert container.upload_id == UPLOAD_ID
+ assert container._headers == EXAMPLE_HEADERS
+ assert container._parts == PARTS
+ assert container._filename == filename
+
+
+def test_xml_mpu_container_initiate(filename):
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL, filename, upload_id=UPLOAD_ID
+ )
+ with pytest.raises(ValueError):
+ container._prepare_initiate_request(BASIC_CONTENT)
+
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL, filename, headers=EXAMPLE_HEADERS
+ )
+ verb, url, body, headers = container._prepare_initiate_request(BASIC_CONTENT)
+ assert verb == _upload._POST
+ assert url == EXAMPLE_XML_UPLOAD_URL + _upload._MPU_INITIATE_QUERY
+ assert not body
+ assert headers == {**EXAMPLE_HEADERS, "content-type": BASIC_CONTENT}
+
+ _fix_up_virtual(container)
+ response = _make_xml_response(
+ text=EXAMPLE_XML_MPU_INITIATE_TEXT_TEMPLATE.format(upload_id=UPLOAD_ID)
+ )
+ container._process_initiate_response(response)
+ assert container.upload_id == UPLOAD_ID
+
+ with pytest.raises(NotImplementedError):
+ container.initiate(None, None)
+
+
+def test_xml_mpu_container_finalize(filename):
+ container = _upload.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename)
+ with pytest.raises(ValueError):
+ container._prepare_finalize_request()
+
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL,
+ filename,
+ headers=EXAMPLE_HEADERS,
+ upload_id=UPLOAD_ID,
+ )
+ container._parts = PARTS
+ verb, url, body, headers = container._prepare_finalize_request()
+ assert verb == _upload._POST
+ final_query = _upload._MPU_FINAL_QUERY_TEMPLATE.format(upload_id=UPLOAD_ID)
+ assert url == EXAMPLE_XML_UPLOAD_URL + final_query
+ assert headers == EXAMPLE_HEADERS
+ assert b"CompleteMultipartUpload" in body
+ for key, value in PARTS.items():
+ assert str(key).encode("utf-8") in body
+ assert value.encode("utf-8") in body
+
+ _fix_up_virtual(container)
+ response = _make_xml_response()
+ container._process_finalize_response(response)
+ assert container.finished
+
+ with pytest.raises(NotImplementedError):
+ container.finalize(None)
+
+
+def test_xml_mpu_container_cancel(filename):
+ container = _upload.XMLMPUContainer(EXAMPLE_XML_UPLOAD_URL, filename)
+ with pytest.raises(ValueError):
+ container._prepare_cancel_request()
+
+ container = _upload.XMLMPUContainer(
+ EXAMPLE_XML_UPLOAD_URL,
+ filename,
+ headers=EXAMPLE_HEADERS,
+ upload_id=UPLOAD_ID,
+ )
+ container._parts = PARTS
+ verb, url, body, headers = container._prepare_cancel_request()
+ assert verb == _upload._DELETE
+ final_query = _upload._MPU_FINAL_QUERY_TEMPLATE.format(upload_id=UPLOAD_ID)
+ assert url == EXAMPLE_XML_UPLOAD_URL + final_query
+ assert headers == EXAMPLE_HEADERS
+ assert not body
+
+ _fix_up_virtual(container)
+ response = _make_xml_response(status_code=204)
+ container._process_cancel_response(response)
+
+ with pytest.raises(NotImplementedError):
+ container.cancel(None)
+
+
+def test_xml_mpu_part(filename):
+ PART_NUMBER = 1
+ START = 0
+ END = 256
+ ETAG = PARTS[1]
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ assert part.upload_url == EXAMPLE_XML_UPLOAD_URL
+ assert part.upload_id == UPLOAD_ID
+ assert part.filename == filename
+ assert part.etag is None
+ assert part.start == START
+ assert part.end == END
+ assert part.part_number == PART_NUMBER
+ assert part._headers == EXAMPLE_HEADERS
+ assert part._checksum_type == "md5"
+ assert part._checksum_object is None
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ )
+ verb, url, payload, headers = part._prepare_upload_request()
+ assert verb == _upload._PUT
+ assert url == EXAMPLE_XML_UPLOAD_URL + _upload._MPU_PART_QUERY_TEMPLATE.format(
+ part=PART_NUMBER, upload_id=UPLOAD_ID
+ )
+ assert headers == EXAMPLE_HEADERS
+ assert payload == FILE_DATA[START:END]
+
+ _fix_up_virtual(part)
+ response = _make_xml_response(headers={"etag": ETAG})
+ part._process_upload_response(response)
+ assert part.etag == ETAG
+
+
+def test_xml_mpu_part_invalid_response(filename):
+ PART_NUMBER = 1
+ START = 0
+ END = 256
+ ETAG = PARTS[1]
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ _fix_up_virtual(part)
+ response = _make_xml_response(headers={"etag": ETAG})
+ with pytest.raises(common.InvalidResponse):
+ part._process_upload_response(response)
+
+
+def test_xml_mpu_part_checksum_failure(filename):
+ PART_NUMBER = 1
+ START = 0
+ END = 256
+ ETAG = PARTS[1]
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ _fix_up_virtual(part)
+ part._prepare_upload_request()
+ response = _make_xml_response(
+ headers={"etag": ETAG, "x-goog-hash": "md5=Ojk9c3dhfxgoKVVHYwFbHQ=="}
+ ) # Example md5 checksum but not the correct one
+ with pytest.raises(common.DataCorruption):
+ part._process_upload_response(response)
+
+
+def test_xml_mpu_part_checksum_success(filename):
+ PART_NUMBER = 1
+ START = 0
+ END = 256
+ ETAG = PARTS[1]
+
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ _fix_up_virtual(part)
+ part._prepare_upload_request()
+ response = _make_xml_response(
+ headers={"etag": ETAG, "x-goog-hash": "md5=pOUFGnohRRFFd24NztFuFw=="}
+ )
+ part._process_upload_response(response)
+ assert part.etag == ETAG
+ assert part.finished
+
+ # Test error handling
+ part = _upload.XMLMPUPart(
+ EXAMPLE_XML_UPLOAD_URL,
+ UPLOAD_ID,
+ filename,
+ START,
+ END,
+ PART_NUMBER,
+ headers=EXAMPLE_HEADERS,
+ checksum="md5",
+ )
+ with pytest.raises(NotImplementedError):
+ part.upload(None)
+ part._finished = True
+ with pytest.raises(ValueError):
+ part._prepare_upload_request()
+
+
+def _make_response(status_code=http.client.OK, headers=None, metadata=None):
+ headers = headers or {}
+ return mock.Mock(
+ headers=headers,
+ status_code=status_code,
+ json=mock.Mock(return_value=metadata),
+ spec=["headers", "status_code"],
+ )
+
+
+def _make_xml_response(status_code=http.client.OK, headers=None, text=None):
+ headers = headers or {}
+ return mock.Mock(
+ headers=headers,
+ status_code=status_code,
+ text=text,
+ spec=["headers", "status_code"],
+ )
+
+
+def _get_status_code(response):
+ return response.status_code
+
+
+def _get_headers(response):
+ return response.headers
+
+
+def _fix_up_virtual(upload):
+ upload._get_status_code = _get_status_code
+ upload._get_headers = _get_headers
+
+
+def _check_retry_strategy(upload):
+ retry_strategy = upload._retry_strategy
+ assert isinstance(retry_strategy, common.RetryStrategy)
+ assert retry_strategy.max_sleep == common.MAX_SLEEP
+ assert retry_strategy.max_cumulative_retry == common.MAX_CUMULATIVE_RETRY
+ assert retry_strategy.max_retries is None
diff --git a/packages/google-resumable-media/tests/unit/test_common.py b/packages/google-resumable-media/tests/unit/test_common.py
new file mode 100644
index 000000000000..d96840c17243
--- /dev/null
+++ b/packages/google-resumable-media/tests/unit/test_common.py
@@ -0,0 +1,85 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from unittest import mock
+import pytest # type: ignore
+
+from google.resumable_media import common
+
+
+class TestInvalidResponse(object):
+ def test_constructor(self):
+ response = mock.sentinel.response
+ error = common.InvalidResponse(response, 1, "a", [b"m"], True)
+
+ assert error.response is response
+ assert error.args == (1, "a", [b"m"], True)
+
+
+class TestRetryStrategy(object):
+ def test_constructor_defaults(self):
+ retry_strategy = common.RetryStrategy()
+ assert retry_strategy.max_sleep == common.MAX_SLEEP
+ assert retry_strategy.max_cumulative_retry == common.MAX_CUMULATIVE_RETRY
+ assert retry_strategy.max_retries is None
+
+ def test_constructor_failure(self):
+ with pytest.raises(ValueError) as exc_info:
+ common.RetryStrategy(max_cumulative_retry=600.0, max_retries=12)
+
+ exc_info.match(common._SLEEP_RETRY_ERROR_MSG)
+
+ def test_constructor_custom_delay_and_multiplier(self):
+ retry_strategy = common.RetryStrategy(initial_delay=3.0, multiplier=4)
+ assert retry_strategy.max_sleep == common.MAX_SLEEP
+ assert retry_strategy.max_cumulative_retry == common.MAX_CUMULATIVE_RETRY
+ assert retry_strategy.max_retries is None
+ assert retry_strategy.initial_delay == 3.0
+ assert retry_strategy.multiplier == 4
+
+ def test_constructor_explicit_bound_cumulative(self):
+ max_sleep = 10.0
+ max_cumulative_retry = 100.0
+ retry_strategy = common.RetryStrategy(
+ max_sleep=max_sleep, max_cumulative_retry=max_cumulative_retry
+ )
+
+ assert retry_strategy.max_sleep == max_sleep
+ assert retry_strategy.max_cumulative_retry == max_cumulative_retry
+ assert retry_strategy.max_retries is None
+
+ def test_constructor_explicit_bound_retries(self):
+ max_sleep = 13.75
+ max_retries = 14
+ retry_strategy = common.RetryStrategy(
+ max_sleep=max_sleep, max_retries=max_retries
+ )
+
+ assert retry_strategy.max_sleep == max_sleep
+ assert retry_strategy.max_cumulative_retry is None
+ assert retry_strategy.max_retries == max_retries
+
+ def test_retry_allowed_bound_cumulative(self):
+ retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0)
+ assert retry_strategy.retry_allowed(50.0, 10)
+ assert retry_strategy.retry_allowed(99.0, 7)
+ assert retry_strategy.retry_allowed(100.0, 4)
+ assert not retry_strategy.retry_allowed(101.0, 11)
+ assert not retry_strategy.retry_allowed(200.0, 6)
+
+ def test_retry_allowed_bound_retries(self):
+ retry_strategy = common.RetryStrategy(max_retries=6)
+ assert retry_strategy.retry_allowed(1000.0, 5)
+ assert retry_strategy.retry_allowed(99.0, 6)
+ assert not retry_strategy.retry_allowed(625.5, 7)
diff --git a/packages/google-resumable-media/tests_async/__init__.py b/packages/google-resumable-media/tests_async/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests_async/data/favicon.ico b/packages/google-resumable-media/tests_async/data/favicon.ico
new file mode 100644
index 000000000000..e9c59160aa3c
Binary files /dev/null and b/packages/google-resumable-media/tests_async/data/favicon.ico differ
diff --git a/packages/google-resumable-media/tests_async/data/file.txt b/packages/google-resumable-media/tests_async/data/file.txt
new file mode 100644
index 000000000000..da07c51074cb
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/data/file.txt
@@ -0,0 +1,64 @@
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
diff --git a/packages/google-resumable-media/tests_async/data/gzipped.txt b/packages/google-resumable-media/tests_async/data/gzipped.txt
new file mode 100644
index 000000000000..da07c51074cb
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/data/gzipped.txt
@@ -0,0 +1,64 @@
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
+abcdefghijklmnopqrstuvwxyz0123456789
diff --git a/packages/google-resumable-media/tests_async/data/gzipped.txt.gz b/packages/google-resumable-media/tests_async/data/gzipped.txt.gz
new file mode 100644
index 000000000000..83e9f396c3c2
Binary files /dev/null and b/packages/google-resumable-media/tests_async/data/gzipped.txt.gz differ
diff --git a/packages/google-resumable-media/tests_async/data/image1.jpg b/packages/google-resumable-media/tests_async/data/image1.jpg
new file mode 100644
index 000000000000..e70137b82794
Binary files /dev/null and b/packages/google-resumable-media/tests_async/data/image1.jpg differ
diff --git a/packages/google-resumable-media/tests_async/data/image2.jpg b/packages/google-resumable-media/tests_async/data/image2.jpg
new file mode 100644
index 000000000000..c3969530e139
Binary files /dev/null and b/packages/google-resumable-media/tests_async/data/image2.jpg differ
diff --git a/packages/google-resumable-media/tests_async/system/__init__.py b/packages/google-resumable-media/tests_async/system/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/system/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests_async/system/credentials.json.enc b/packages/google-resumable-media/tests_async/system/credentials.json.enc
new file mode 100644
index 000000000000..19e26ade73f1
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/system/credentials.json.enc
@@ -0,0 +1,52 @@
+U2FsdGVkX1+wqu1+eVu6OPbPoE0lzIp3B11p8Rdbha1ukxXcsskegJdBjcUqQOav
+W2N3vhA7YfXW/F3T+tZMYYWk5a0vAjxLov3MgFfhvGPK0UzDwKNIXRgxhcLjcSeQ
+ZmSN2kqpmSSKEPLxP0B6r50nAG6r8NYbZWs02lH2e3NGbsoGgP5PQV2oP/ZVYkET
+qABgSd+xkOjE/7664QRfs/5Jl3Pl045Mzl87l1kN6oeoFpxeFqGWOR4WNflauS3s
+96SKsbrCQ4aF/9n9hCz31J9cJosu54eTB9s0fKBkDx7xmouwT3Cqv2KGwJPUCRHk
+3a+3ijxhNz65dYCRp20dUpJuudFQvMpsptn7oAFtNQhvcFrpjnyBn3ODr9JhLBEy
+PTdJbv06ufb+SH9YNMpH3nTYCkS7ZgrnzhteFJtoMzX6sAYiMUmIZtGY7J8MaSE0
+AYqTO/EGkzzSw33o2nNGcg0lsW1tdmY5GKuJ3jlc1Hi6RHpmgbdv+0dAYi734sYs
++0wE18QMe4/RIOCBslMAWvlo9LX9QDLkolToToQ+HN/kJNQOumkxwcjBV3piiJQH
+LaX9bI6lnqkoMl/2GvuR+oQTfzQxjGKdenLWZO2ODH2rr90hXi9vlXjdpDGreMGy
+Mv4lcwmw3Pd1JreKJtdc2ObDrU/o7wDJe4txNCGwCSAZacI+5c/27mT1yOfgE/EK
+Q3LHjqZhFlLI4K0KqH+dyQutL7b1uPtQpeWAVAt/yHs7nNWF62UAdVR+hZyko2Dy
+HWoYtJDMazfpS98c8VWi0FyGfYVESedWvBCLHch4wWqaccY0HWk9sehyC4XrPX8v
+OMw6J1va3vprzCQte56fXNzzpU6f0XeT3OGj5RCN/POMnN+cjyuwqFOsWNCfpXaV
+lhNj3zg+fMk4mM+wa2KdUk6xa0vj7YblgJ5uvZ3lG81ydZCRoFWqaO6497lnj8NV
+SEDqDdJ+/dw+Sf2ur3hyJ9DW0JD8QJkSwfLrqT51eoOqTfFFGdwy2iuXP426l/NH
+mkyusp8UZNPaKZSF9jC8++18fC2Nbbd+dTIn6XWdZKKRZLZ/hca8QP0QesrtYo36
+6kx8Kl3nAbgOk9wFFsZdkUyOy3iRxkBF0qoaH1kPzyxIpNeeIg5cBPWLwN5FVBdd
+eBy8R4i4y/W8yhib34vcOliP0IfAB/VvXJRMUCc1bENfZskMb4mvtsYblyf68Fne
+OjtcSKV2drO+mRmH1H2sPH/yE2yVDivhY5FJxDRFMnS9HXDMpGoukirMLgCjnSre
+ZXMVaDzkRw1RtsOms+F7EVJb5v/HKu6I34YNJDlAFy6AASmz+H0EXBDK4mma8GSu
+BOgPY3PbF8R+KnzKsOVbaOon90dGclnUNlqnVvsnNeWWKJmL7rCPkMHfb5dBhw60
+j9oLmu74+xmuf9aqzSvrcaHV9u+zf2eCsdQJhttaDYFAKg1q43fhZYHIaURidoD+
+UTxn0AVygiKkTwTFQl1+taDiRffOtNvumSLZG9n8cimoBvzKle3H9tv43uyO6muG
+ty0m8Pyk5LyLE9DaDQwxq+++8g7boXQe7jCtAIMxRveIdwWPI/XHbyZ3I4uTG65F
+RV5K8Q34VVjagdPMNq0ijo73iYy5RH18MSQc8eG3UtqVvr/QeSdPEb8N6o+OwEG8
+VuAFbKPHMfQrjwGCtr0YvHTmvZPlFef+J3iH6WPfFFbe5ZS8XQUoR1dZHX9BXIXK
+Om/itKUoHvAuYIqjTboqK181OVr/9a2FipXxbenXYiWXRtLGpHeetZbKRhxwWe0h
+kDdDL/XglsRNasfLz4c9AyGzJJi7J9Pr7uBSX9QFHLeGQP6jfHrEqBkiGEUP9iQr
+11wabtNouC+1tT0erBAm/KEps81l76NZ7OxqOM8mLrdAE8RO/ypZTqZW4saQnry/
+iUGhwEnRNZpEh8xiYSZ8JgUTbbKo4+FXZxUwV1DBQ7oroPrduaukd68m4E6Tqsx+
+lTl25hLhNTEJCYQ0hg2CeZdSpOPGgpn+zhLDvlQ0lPZDCByh9xCepAq/oUArddln
+vobPdBRVW27gYntAYMlFbc1hSN/LKoZOYq6jBNAPykiv5tTWNV71HUE7b1nRfo27
+aGf3Ptzu7GRXVLom+WKxswUqzkWC8afvrNnZ040wiLQnWzn2yxytipUg3UxIvP+U
+klWj8Tt1wBmG/JGLEThwcjPTOGvDkocQAAImlV3diiqwTHlj+pLZVRtJA4SOQxI8
+ChFi73B8gPOexfqYPUFdB90FJWsxTQGZaucyuNTqFMuJ9eEDP5WmK4lcJuKFTCGT
+M4VYd9j4JlxRRQxKkMhfoXeUsW3TH6uAmKxN79AiYnOh6QUIv+PP+yt9WwQhNqkb
+7otLl0AKdMBizxyq6AExlw/VmdYDJxcZ4Y/P+M85Ae5e+Lz/XjWHLnjP1BPI6C+n
+A/RbICOd/W/wf6ZOZlVBW1wePv0M5jWDGL086lHVrgBnzdWrQTHhzG43v1IaN/vK
+EVZfvkqTe5AWNoK1Da/zEafWf0jzc4cS0grCA9KJ0nHwRYYEG0YQAGqY12PDn9tH
+WjCVDa6wlw/Niq6BAmkE8d9ds2I8l0Xm1eHaMM3U3xY0OsmDYVP2p+BXZ7qWKa9c
+XjuT8gWTS0gZqerlALxTsIEy4/5iKhqdepjAefZxozS30kZhCMG7WXORV9pcdYFP
+rCoVPES85sAfwjjL9ZxmtoqH5845KoTlZWqbI/NJ/KCNa1VGXcc7NuNnCUo8sWqe
+kTwFSOnF+kaXtDFjM5/7/eQWKBelWWXysMX2+pUCQdIcUa5LW3M+16AjF906+DGZ
+pptUebilOd7CEXFKwgO2dZXLkTXj5hyKHYyTt066jPIdyAfGZe9oF0ttzwSS74WY
+Y1Sx1PvAH8B5+jfGnYKhVZHbX0nzdBvwG3FNlg2+GVrpTynTH1l1pVUV8YWrbWhh
+JE+xjLk0RKfC9jmhs3EenpfpYAEkIKZO3CGVXhZMi4kd7wUZud9vGjOcBlOF3YGG
+cVjYDRAymlY1VH3hvkToMZPdjJk8+1fT0bbWTXXjppV3tpC9aybz4H3BOvTXh8MN
+c7X4Pn1rDgjtPK2HfvuR6t9+LqWYTM15NeTnEtdkDdQGUmr3CYQI2h07bQYjtGDY
+XCfYZ4rRLYGcXiRKmm+NGGb/rsJcJe0KeVPZZmIFP5gfvmWvaQeY4lYw1YABdh9Y
+gTIqd+T4OGB5S9EIGrG6uXrlJkCZnIxOJjBPGkVsygn2QOdkIJ8tnycXB3ChTBfL
+FMA3i59W/pGf9apHpGF+iA==
diff --git a/packages/google-resumable-media/tests_async/system/requests/__init__.py b/packages/google-resumable-media/tests_async/system/requests/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/system/requests/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests_async/system/requests/conftest.py b/packages/google-resumable-media/tests_async/system/requests/conftest.py
new file mode 100644
index 000000000000..81c39df2ecfe
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/system/requests/conftest.py
@@ -0,0 +1,61 @@
+# Copyright 2019 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""py.test fixtures to be shared across multiple system test modules."""
+
+from tests.system import utils
+
+from google.auth._default_async import default_async # type: ignore
+import google.auth.transport._aiohttp_requests as tr_requests # type: ignore
+import pytest # type: ignore
+
+
+async def ensure_bucket(transport):
+ get_response = await transport.request("GET", utils.BUCKET_URL)
+ if get_response.status == 404:
+ credentials = transport.credentials
+ query_params = {"project": credentials.project_id}
+ payload = {"name": utils.BUCKET_NAME}
+ post_response = await transport.request(
+ "POST", utils.BUCKET_POST_URL, params=query_params, json=payload
+ )
+ if not (post_response.status == 200):
+ raise ValueError(
+ "{}: {}".format(post_response.status, post_response.reason)
+ )
+
+
+async def cleanup_bucket(transport):
+ del_response = await transport.request("DELETE", utils.BUCKET_URL)
+
+ if not (del_response.status == 204):
+ raise ValueError("{}: {}".format(del_response.status, del_response.reason))
+
+
+def _get_authorized_transport():
+ credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,))
+ return tr_requests.AuthorizedSession(credentials)
+
+
+@pytest.fixture(scope="module")
+async def authorized_transport():
+ credentials, project_id = default_async(scopes=(utils.GCS_RW_SCOPE,))
+ yield _get_authorized_transport()
+
+
+@pytest.fixture(scope="session")
+async def bucket():
+ authorized_transport = _get_authorized_transport()
+ await ensure_bucket(authorized_transport)
+ yield utils.BUCKET_URL
+ await cleanup_bucket(authorized_transport)
diff --git a/packages/google-resumable-media/tests_async/system/requests/test_download.py b/packages/google-resumable-media/tests_async/system/requests/test_download.py
new file mode 100644
index 000000000000..483d8598698d
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/system/requests/test_download.py
@@ -0,0 +1,608 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import copy
+import hashlib
+import http.client
+import io
+import os
+
+import asyncio
+from google.auth._default_async import default_async # type: ignore
+import google.auth.transport._aiohttp_requests as tr_requests # type: ignore
+import multidict # type: ignore
+import pytest # type: ignore
+
+import google._async_resumable_media.requests as resumable_requests
+from google.resumable_media import _helpers
+import google._async_resumable_media.requests.download as download_mod
+from google.resumable_media import common
+from tests.system import utils
+
+
+CURR_DIR = os.path.dirname(os.path.realpath(__file__))
+DATA_DIR = os.path.join(CURR_DIR, "..", "..", "data")
+PLAIN_TEXT = "text/plain"
+IMAGE_JPEG = "image/jpeg"
+ENCRYPTED_ERR = b"The target object is encrypted by a customer-supplied encryption key."
+NO_BODY_ERR = "The content for this response was already consumed"
+NOT_FOUND_ERR = (
+ b"No such object: " + utils.BUCKET_NAME.encode("utf-8") + b"/does-not-exist.txt"
+)
+SIMPLE_DOWNLOADS = (resumable_requests.Download, resumable_requests.RawDownload)
+
+
+@pytest.fixture(scope="session")
+def event_loop(request):
+ """Create an instance of the default event loop for each test session."""
+ loop = asyncio.get_event_loop_policy().new_event_loop()
+ yield loop
+ loop.close()
+
+
+class CorruptingAuthorizedSession(tr_requests.AuthorizedSession):
+ """A Requests Session class with credentials, which corrupts responses.
+
+ This class is used for testing checksum validation.
+
+ Args:
+ credentials (google.auth.credentials.Credentials): The credentials to
+ add to the request.
+ refresh_status_codes (Sequence[int]): Which HTTP status codes indicate
+ that credentials should be refreshed and the request should be
+ retried.
+ max_refresh_attempts (int): The maximum number of times to attempt to
+ refresh the credentials and retry the request.
+ kwargs: Additional arguments passed to the :class:`requests.Session`
+ constructor.
+ """
+
+ EMPTY_MD5 = base64.b64encode(hashlib.md5(b"").digest()).decode("utf-8")
+ crc32c = _helpers._get_crc32c_object()
+ crc32c.update(b"")
+ EMPTY_CRC32C = base64.b64encode(crc32c.digest()).decode("utf-8")
+
+ async def request(self, method, url, data=None, headers=None, **kwargs):
+ """Implementation of Requests' request."""
+ response = await tr_requests.AuthorizedSession.request(
+ self, method, url, data=data, headers=headers, **kwargs
+ )
+
+ temp = multidict.CIMultiDict(response.headers)
+ temp[_helpers._HASH_HEADER] = "crc32c={},md5={}".format(
+ self.EMPTY_CRC32C, self.EMPTY_MD5
+ )
+ response._headers = temp
+
+ return response
+
+
+def get_path(filename):
+ return os.path.realpath(os.path.join(DATA_DIR, filename))
+
+
+ALL_FILES = (
+ {
+ "path": get_path("image1.jpg"),
+ "content_type": IMAGE_JPEG,
+ "md5": "1bsd83IYNug8hd+V1ING3Q==",
+ "crc32c": "YQGPxA==",
+ "slices": (
+ slice(1024, 16386, None), # obj[1024:16386]
+ slice(None, 8192, None), # obj[:8192]
+ slice(-256, None, None), # obj[-256:]
+ slice(262144, None, None), # obj[262144:]
+ ),
+ },
+ {
+ "path": get_path("image2.jpg"),
+ "content_type": IMAGE_JPEG,
+ "md5": "gdLXJltiYAMP9WZZFEQI1Q==",
+ "crc32c": "sxxEFQ==",
+ "slices": (
+ slice(1024, 16386, None), # obj[1024:16386]
+ slice(None, 8192, None), # obj[:8192]
+ slice(-256, None, None), # obj[-256:]
+ slice(262144, None, None), # obj[262144:]
+ ),
+ },
+ {
+ "path": get_path("file.txt"),
+ "content_type": PLAIN_TEXT,
+ "md5": "XHSHAr/SpIeZtZbjgQ4nGw==",
+ "crc32c": "MeMHoQ==",
+ "slices": (),
+ },
+ {
+ "path": get_path("gzipped.txt.gz"),
+ "uncompressed": get_path("gzipped.txt"),
+ "content_type": PLAIN_TEXT,
+ "md5": "KHRs/+ZSrc/FuuR4qz/PZQ==",
+ "crc32c": "/LIRNg==",
+ "slices": (),
+ "metadata": {"contentEncoding": "gzip"},
+ },
+)
+
+
+def get_contents_for_upload(info):
+ with open(info["path"], "rb") as file_obj:
+ return file_obj.read()
+
+
+def get_contents(info):
+ full_path = info.get("uncompressed", info["path"])
+ with open(full_path, "rb") as file_obj:
+ return file_obj.read()
+
+
+def get_raw_contents(info):
+ full_path = info["path"]
+ with open(full_path, "rb") as file_obj:
+ return file_obj.read()
+
+
+def get_blob_name(info):
+ full_path = info.get("uncompressed", info["path"])
+ return os.path.basename(full_path)
+
+
+async def delete_blob(transport, blob_name):
+ metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
+ response = await transport.request("DELETE", metadata_url)
+ assert response.status == http.client.NO_CONTENT
+
+
+@pytest.fixture(scope="module")
+async def secret_file(authorized_transport, bucket):
+ blob_name = "super-seekrit.txt"
+ data = b"Please do not tell anyone my encrypted seekrit."
+
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ headers = utils.get_encryption_headers()
+ upload = resumable_requests.SimpleUpload(upload_url, headers=headers)
+ response = await upload.transmit(authorized_transport, data, PLAIN_TEXT)
+ assert response.status == http.client.OK
+
+ yield blob_name, data, headers
+
+ await delete_blob(authorized_transport, blob_name)
+
+
+# Transport that returns corrupt data, so we can exercise checksum handling.
+@pytest.fixture(scope="module")
+async def corrupting_transport():
+ credentials, _ = default_async(scopes=(utils.GCS_RW_SCOPE,))
+ yield CorruptingAuthorizedSession(credentials)
+
+
+@pytest.fixture(scope="module")
+async def simple_file(authorized_transport, bucket):
+ blob_name = "basic-file.txt"
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ upload = resumable_requests.SimpleUpload(upload_url)
+ data = b"Simple contents"
+ response = await upload.transmit(authorized_transport, data, PLAIN_TEXT)
+ assert response.status == http.client.OK
+
+ yield blob_name, data
+
+ await delete_blob(authorized_transport, blob_name)
+
+
+@pytest.fixture(scope="module")
+async def add_files(authorized_transport, bucket):
+ blob_names = []
+ for info in ALL_FILES:
+ to_upload = get_contents_for_upload(info)
+ blob_name = get_blob_name(info)
+
+ blob_names.append(blob_name)
+ if "metadata" in info:
+ upload = resumable_requests.MultipartUpload(utils.MULTIPART_UPLOAD)
+ metadata = copy.deepcopy(info["metadata"])
+ metadata["name"] = blob_name
+ response = await upload.transmit(
+ authorized_transport, to_upload, metadata, info["content_type"]
+ )
+ else:
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ upload = resumable_requests.SimpleUpload(upload_url)
+ response = await upload.transmit(
+ authorized_transport, to_upload, info["content_type"]
+ )
+
+ assert response.status == http.client.OK
+
+ yield
+
+ # Clean-up the blobs we created.
+ for blob_name in blob_names:
+ await delete_blob(authorized_transport, blob_name)
+
+
+async def check_tombstoned(download, transport):
+ assert download.finished
+ if isinstance(download, SIMPLE_DOWNLOADS):
+ with pytest.raises(ValueError) as exc_info:
+ await download.consume(transport)
+ assert exc_info.match("A download can only be used once.")
+ else:
+ with pytest.raises(ValueError) as exc_info:
+ await download.consume_next_chunk(transport)
+ assert exc_info.match("Download has finished.")
+
+
+async def check_error_response(exc_info, status_code, message):
+ error = exc_info.value
+ response = error.response
+ assert response.status == status_code
+ content = await response.content.read()
+ assert content.startswith(message)
+ assert len(error.args) == 5
+ assert error.args[1] == status_code
+ assert error.args[3] == http.client.OK
+ assert error.args[4] == http.client.PARTIAL_CONTENT
+
+
+class TestDownload(object):
+ @staticmethod
+ def _get_target_class():
+ return resumable_requests.Download
+
+ def _make_one(self, media_url, **kw):
+ return self._get_target_class()(media_url, **kw)
+
+ @staticmethod
+ def _get_contents(info):
+ return get_contents(info)
+
+ @staticmethod
+ async def _read_response_content(response):
+ content = await response.content()
+ return content
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ async def test_download_full(self, add_files, authorized_transport, checksum):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = self._make_one(media_url, checksum=checksum)
+ # Consume the resource.
+ response = await download.consume(authorized_transport)
+ response = tr_requests._CombinedResponse(response)
+ assert response.status == http.client.OK
+ content = await self._read_response_content(response)
+ assert content == actual_contents
+ await check_tombstoned(download, authorized_transport)
+
+ @pytest.mark.asyncio
+ async def test_extra_headers(self, authorized_transport, secret_file):
+ blob_name, data, headers = secret_file
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = self._make_one(media_url, headers=headers)
+ # Consume the resource.
+ response = await download.consume(authorized_transport)
+ assert response.status == http.client.OK
+ content = await response.content.read()
+ assert content == data
+ await check_tombstoned(download, authorized_transport)
+
+ # Attempt to consume the resource **without** the headers.
+
+ download_wo = self._make_one(media_url)
+
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await download_wo.consume(authorized_transport)
+
+ await check_error_response(exc_info, http.client.BAD_REQUEST, ENCRYPTED_ERR)
+ await check_tombstoned(download_wo, authorized_transport)
+
+ @pytest.mark.asyncio
+ async def test_non_existent_file(self, authorized_transport, bucket):
+ blob_name = "does-not-exist.txt"
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = self._make_one(media_url)
+
+ # Try to consume the resource and fail.
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await download.consume(authorized_transport)
+ await check_error_response(exc_info, http.client.NOT_FOUND, NOT_FOUND_ERR)
+ await check_tombstoned(download, authorized_transport)
+
+ @pytest.mark.asyncio
+ async def test_bad_range(self, simple_file, authorized_transport):
+ blob_name, data = simple_file
+ # Make sure we have an invalid range.
+ start = 32
+ end = 63
+ assert len(data) < start < end
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = self._make_one(media_url, start=start, end=end)
+
+ # Try to consume the resource and fail.
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await download.consume(authorized_transport)
+
+ await check_error_response(
+ exc_info,
+ http.client.REQUESTED_RANGE_NOT_SATISFIABLE,
+ b"Request range not satisfiable",
+ )
+ await check_tombstoned(download, authorized_transport)
+
+ def _download_slice(self, media_url, slice_):
+ assert slice_.step is None
+
+ end = None
+ if slice_.stop is not None:
+ end = slice_.stop - 1
+
+ return self._make_one(media_url, start=slice_.start, end=end)
+
+ @pytest.mark.asyncio
+ async def test_download_partial(self, add_files, authorized_transport):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ for slice_ in info["slices"]:
+ download = self._download_slice(media_url, slice_)
+ response = await download.consume(authorized_transport)
+ assert response.status == http.client.PARTIAL_CONTENT
+ content = await response.content.read()
+ assert content == actual_contents[slice_]
+ with pytest.raises(ValueError):
+ await download.consume(authorized_transport)
+
+
+class TestRawDownload(TestDownload):
+ @staticmethod
+ def _get_target_class():
+ return resumable_requests.RawDownload
+
+ @staticmethod
+ def _get_contents(info):
+ return get_raw_contents(info)
+
+ @staticmethod
+ async def _read_response_content(response):
+ content = await tr_requests._CombinedResponse(response._response).raw_content()
+ return content
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test_corrupt_download(self, add_files, corrupting_transport, checksum):
+ for info in ALL_FILES:
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, stream=stream, checksum=checksum)
+ # Consume the resource.
+ with pytest.raises(common.DataCorruption) as exc_info:
+ await download.consume(corrupting_transport)
+
+ assert download.finished
+
+ if checksum == "md5":
+ EMPTY_HASH = CorruptingAuthorizedSession.EMPTY_MD5
+ else:
+ EMPTY_HASH = CorruptingAuthorizedSession.EMPTY_CRC32C
+
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ download.media_url,
+ EMPTY_HASH,
+ info[checksum],
+ checksum_type=checksum.upper(),
+ )
+ assert exc_info.value.args == (msg,)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test_corrupt_download_no_check(
+ self, add_files, corrupting_transport, checksum
+ ):
+ for info in ALL_FILES:
+ blob_name = get_blob_name(info)
+
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, stream=stream, checksum=None)
+ # Consume the resource.
+ await download.consume(corrupting_transport)
+
+ assert download.finished
+
+
+def get_chunk_size(min_chunks, total_bytes):
+ # Make sure the number of chunks **DOES NOT** evenly divide.
+ num_chunks = min_chunks
+ while total_bytes % num_chunks == 0:
+ num_chunks += 1
+
+ chunk_size = total_bytes // num_chunks
+ # Since we know an integer division has remainder, increment by 1.
+ chunk_size += 1
+ assert total_bytes < num_chunks * chunk_size
+
+ return num_chunks, chunk_size
+
+
+async def consume_chunks(download, authorized_transport, total_bytes, actual_contents):
+ start_byte = download.start
+ end_byte = download.end
+ if end_byte is None:
+ end_byte = total_bytes - 1
+
+ num_responses = 0
+ while not download.finished:
+ response = await download.consume_next_chunk(authorized_transport)
+ num_responses += 1
+
+ next_byte = min(start_byte + download.chunk_size, end_byte + 1)
+ assert download.bytes_downloaded == next_byte - download.start
+ assert download.total_bytes == total_bytes
+ assert response.status == http.client.PARTIAL_CONTENT
+ # NOTE: Due to the consumption of the stream in the respone, the
+ # response object for async requests will be EOF at this point. In
+ # sync versions we could compare the content with the range of
+ # actual contents. Since streams aren't reversible, we can't do that
+ # here.
+ assert response.content.at_eof()
+
+ start_byte = next_byte
+
+ return num_responses, response
+
+
+class TestChunkedDownload(object):
+ @staticmethod
+ def _get_target_class():
+ return resumable_requests.ChunkedDownload
+
+ def _make_one(self, media_url, chunk_size, stream, **kw):
+ return self._get_target_class()(media_url, chunk_size, stream, **kw)
+
+ @staticmethod
+ def _get_contents(info):
+ return get_contents(info)
+
+ @pytest.mark.asyncio
+ async def test_chunked_download_partial(self, add_files, authorized_transport):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ for slice_ in info["slices"]:
+ # Manually replace a missing start with 0.
+ start = 0 if slice_.start is None else slice_.start
+ # Chunked downloads don't support a negative index.
+ if start < 0:
+ continue
+
+ # First determine how much content is in the slice and
+ # use it to determine a chunking strategy.
+ total_bytes = len(actual_contents)
+ if slice_.stop is None:
+ end_byte = total_bytes - 1
+ end = None
+ else:
+ # Python slices DO NOT include the last index, though a byte
+ # range **is** inclusive of both endpoints.
+ end_byte = slice_.stop - 1
+ end = end_byte
+
+ num_chunks, chunk_size = get_chunk_size(7, end_byte - start + 1)
+ # Create the actual download object.
+ stream = io.BytesIO()
+ download = self._make_one(
+ media_url, chunk_size, stream, start=start, end=end
+ )
+ # Consume the resource in chunks.
+ num_responses, last_response = await consume_chunks(
+ download, authorized_transport, total_bytes, actual_contents
+ )
+
+ # Make sure the combined chunks are the whole slice.
+ assert stream.getvalue() == actual_contents[slice_]
+ # Check that we have the right number of responses.
+ assert num_responses == num_chunks
+ # Make sure the last chunk isn't the same size.
+ content = await last_response.content.read()
+ assert len(content) < chunk_size
+ await check_tombstoned(download, authorized_transport)
+
+ @pytest.mark.asyncio
+ async def test_chunked_with_extra_headers(self, authorized_transport, secret_file):
+ blob_name, data, headers = secret_file
+ num_chunks = 4
+ chunk_size = 12
+ assert (num_chunks - 1) * chunk_size < len(data) < num_chunks * chunk_size
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, chunk_size, stream, headers=headers)
+ # Consume the resource in chunks.
+ num_responses, last_response = await consume_chunks(
+ download, authorized_transport, len(data), data
+ )
+ # Make sure the combined chunks are the whole object.
+ assert stream.getvalue() == data
+ # Check that we have the right number of responses.
+ assert num_responses == num_chunks
+ # Make sure the last chunk isn't the same size.
+
+ content = await last_response.read()
+ assert len(content) < chunk_size
+
+ await check_tombstoned(download, authorized_transport)
+ # Attempt to consume the resource **without** the headers.
+ stream_wo = io.BytesIO()
+ download_wo = resumable_requests.ChunkedDownload(
+ media_url, chunk_size, stream_wo
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await download_wo.consume_next_chunk(authorized_transport)
+
+ assert stream_wo.tell() == 0
+ await check_error_response(exc_info, http.client.BAD_REQUEST, ENCRYPTED_ERR)
+ assert download_wo.invalid
+
+
+class TestRawChunkedDownload(TestChunkedDownload):
+ @staticmethod
+ def _get_target_class():
+ return resumable_requests.RawChunkedDownload
+
+ @staticmethod
+ def _get_contents(info):
+ return get_raw_contents(info)
+
+ @pytest.mark.asyncio
+ async def test_chunked_download_full(self, add_files, authorized_transport):
+ for info in ALL_FILES:
+ actual_contents = self._get_contents(info)
+ blob_name = get_blob_name(info)
+
+ total_bytes = len(actual_contents)
+ num_chunks, chunk_size = get_chunk_size(7, total_bytes)
+ # Create the actual download object.
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ stream = io.BytesIO()
+ download = self._make_one(media_url, chunk_size, stream)
+ # Consume the resource in chunks.
+ num_responses, last_response = await consume_chunks(
+ download, authorized_transport, total_bytes, actual_contents
+ )
+ # Make sure the combined chunks are the whole object.
+ assert stream.getvalue() == actual_contents
+ # Check that we have the right number of responses.
+ assert num_responses == num_chunks
+ # Make sure the last chunk isn't the same size.
+ assert total_bytes % chunk_size != 0
+ content = await last_response.content.read()
+ assert len(content) < chunk_size
+ await check_tombstoned(download, authorized_transport)
diff --git a/packages/google-resumable-media/tests_async/system/requests/test_upload.py b/packages/google-resumable-media/tests_async/system/requests/test_upload.py
new file mode 100644
index 000000000000..fb8ba51a6aa7
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/system/requests/test_upload.py
@@ -0,0 +1,698 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import hashlib
+import http.client
+import io
+import os
+import urllib.parse
+
+import asyncio
+import mock
+import pytest # type: ignore
+
+from google.resumable_media import common
+from google import _async_resumable_media
+import google._async_resumable_media.requests as resumable_requests
+from google.resumable_media import _helpers
+from tests.system import utils
+
+
+CURR_DIR = os.path.dirname(os.path.realpath(__file__))
+DATA_DIR = os.path.join(CURR_DIR, "..", "..", "data")
+ICO_FILE = os.path.realpath(os.path.join(DATA_DIR, "favicon.ico"))
+IMAGE_FILE = os.path.realpath(os.path.join(DATA_DIR, "image1.jpg"))
+ICO_CONTENT_TYPE = "image/x-icon"
+JPEG_CONTENT_TYPE = "image/jpeg"
+BYTES_CONTENT_TYPE = "application/octet-stream"
+BAD_CHUNK_SIZE_MSG = (
+ b"Invalid request. The number of bytes uploaded is required to be equal "
+ b"or greater than 262144, except for the final request (it's recommended "
+ b"to be the exact multiple of 262144). The received request contained "
+ b"1024 bytes, which does not meet this requirement."
+)
+
+
+@pytest.fixture(scope="session")
+def event_loop(request):
+ """Create an instance of the default event loop for each test session."""
+ loop = asyncio.get_event_loop_policy().new_event_loop()
+ yield loop
+ loop.close()
+
+
+@pytest.fixture
+async def cleanup():
+ to_delete = []
+
+ async def add_cleanup(blob_name, transport):
+ to_delete.append((blob_name, transport))
+
+ yield add_cleanup
+
+ for blob_name, transport in to_delete:
+ metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
+ response = await transport.request("DELETE", metadata_url)
+ assert response.status == http.client.NO_CONTENT
+
+
+@pytest.fixture
+def img_stream():
+ """Open-file as a fixture.
+
+ This is so that an entire test can execute in the context of
+ the context manager without worrying about closing the file.
+ """
+ with open(IMAGE_FILE, "rb") as file_obj:
+ yield file_obj
+
+
+def get_md5(data):
+ hash_obj = hashlib.md5(data)
+ return base64.b64encode(hash_obj.digest())
+
+
+def get_upload_id(upload_url):
+ parse_result = urllib.parse.urlparse(upload_url)
+ parsed_query = urllib.parse.parse_qs(parse_result.query)
+ # NOTE: We are unpacking here, so asserting exactly one match.
+ (upload_id,) = parsed_query["upload_id"]
+ return upload_id
+
+
+def get_num_chunks(total_bytes, chunk_size):
+ expected_chunks, remainder = divmod(total_bytes, chunk_size)
+ if remainder > 0:
+ expected_chunks += 1
+ return expected_chunks
+
+
+async def check_response(
+ response,
+ blob_name,
+ actual_contents=None,
+ total_bytes=None,
+ metadata=None,
+ content_type=ICO_CONTENT_TYPE,
+):
+ assert response.status == http.client.OK
+
+ json_response = await response.json()
+ assert json_response["bucket"] == utils.BUCKET_NAME
+ assert json_response["contentType"] == content_type
+ if actual_contents is not None:
+ md5_hash = json_response["md5Hash"].encode("ascii")
+ assert md5_hash == get_md5(actual_contents)
+ total_bytes = len(actual_contents)
+ assert json_response["metageneration"] == "1"
+ assert json_response["name"] == blob_name
+ assert json_response["size"] == "{:d}".format(total_bytes)
+ assert json_response["storageClass"] == "STANDARD"
+ if metadata is None:
+ assert "metadata" not in json_response
+ else:
+ assert json_response["metadata"] == metadata
+
+
+async def check_content(blob_name, expected_content, transport, headers=None):
+ media_url = utils.DOWNLOAD_URL_TEMPLATE.format(blob_name=blob_name)
+ download = resumable_requests.Download(media_url, headers=headers)
+ response = await download.consume(transport)
+ content = await response.content.read()
+ assert response.status == http.client.OK
+ assert content == expected_content
+
+
+async def check_tombstoned(upload, transport, *args):
+ assert upload.finished
+ basic_types = (resumable_requests.SimpleUpload, resumable_requests.MultipartUpload)
+
+ if isinstance(upload, basic_types):
+ with pytest.raises(ValueError):
+ await upload.transmit(transport, *args)
+ else:
+ with pytest.raises(ValueError):
+ await upload.transmit_next_chunk(transport, *args)
+
+
+async def check_does_not_exist(transport, blob_name):
+ metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
+ # Make sure we are creating a **new** object.
+ response = await transport.request("GET", metadata_url)
+ assert response.status == http.client.NOT_FOUND
+
+
+async def check_initiate(response, upload, stream, transport, metadata):
+ assert response.status == http.client.OK
+ content = await response.content.read()
+ assert content == b""
+ upload_id = get_upload_id(upload.resumable_url)
+ assert response.headers["x-guploader-uploadid"] == upload_id
+ assert stream.tell() == 0
+ # Make sure the upload cannot be re-initiated.
+ with pytest.raises(ValueError) as exc_info:
+ await upload.initiate(transport, stream, metadata, JPEG_CONTENT_TYPE)
+
+ exc_info.match("This upload has already been initiated.")
+
+
+async def check_bad_chunk(upload, transport):
+ with pytest.raises(_async_resumable_media.InvalidResponse) as exc_info:
+ await upload.transmit_next_chunk(transport)
+ error = exc_info.value
+ response = error.response
+ assert response.status == http.client.BAD_REQUEST
+ content = await response.content.read()
+ assert content == BAD_CHUNK_SIZE_MSG
+
+
+async def transmit_chunks(
+ upload, transport, blob_name, metadata, num_chunks=0, content_type=JPEG_CONTENT_TYPE
+):
+ while not upload.finished:
+ num_chunks += 1
+ response = await upload.transmit_next_chunk(transport)
+ if upload.finished:
+ assert upload.bytes_uploaded == upload.total_bytes
+ await check_response(
+ response,
+ blob_name,
+ total_bytes=upload.total_bytes,
+ metadata=metadata,
+ content_type=content_type,
+ )
+ else:
+ assert upload.bytes_uploaded == num_chunks * upload.chunk_size
+ assert response.status == http.client.PERMANENT_REDIRECT
+
+ return num_chunks
+
+
+@pytest.mark.asyncio
+async def test_simple_upload(authorized_transport, bucket, cleanup):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ # Make sure to clean up the uploaded blob when we are done.
+
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ upload = resumable_requests.SimpleUpload(upload_url)
+ # Transmit the resource.
+ response = await upload.transmit(
+ authorized_transport, actual_contents, ICO_CONTENT_TYPE
+ )
+ await check_response(response, blob_name, actual_contents=actual_contents)
+ # Download the content to make sure it's "working as expected".
+ await check_content(blob_name, actual_contents, authorized_transport)
+ # Make sure the upload is tombstoned.
+ await check_tombstoned(
+ upload, authorized_transport, actual_contents, ICO_CONTENT_TYPE
+ )
+
+
+@pytest.mark.asyncio
+async def test_simple_upload_with_headers(authorized_transport, bucket, cleanup):
+ blob_name = "some-stuff.bin"
+ # Make sure to clean up the uploaded blob when we are done.
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.SIMPLE_UPLOAD_TEMPLATE.format(blob_name=blob_name)
+ headers = utils.get_encryption_headers()
+ upload = resumable_requests.SimpleUpload(upload_url, headers=headers)
+ # Transmit the resource.
+ data = b"Binary contents\x00\x01\x02."
+ response = await upload.transmit(authorized_transport, data, BYTES_CONTENT_TYPE)
+ await check_response(
+ response, blob_name, actual_contents=data, content_type=BYTES_CONTENT_TYPE
+ )
+ # Download the content to make sure it's "working as expected".
+ await check_content(blob_name, data, authorized_transport, headers=headers)
+ # Make sure the upload is tombstoned.
+ await check_tombstoned(upload, authorized_transport, data, BYTES_CONTENT_TYPE)
+
+
+@pytest.mark.asyncio
+async def test_multipart_upload(authorized_transport, bucket, cleanup):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ # Make sure to clean up the uploaded blob when we are done.
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.MULTIPART_UPLOAD
+ upload = resumable_requests.MultipartUpload(upload_url)
+ # Transmit the resource.
+ metadata = {"name": blob_name, "metadata": {"color": "yellow"}}
+ response = await upload.transmit(
+ authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE
+ )
+ await check_response(
+ response,
+ blob_name,
+ actual_contents=actual_contents,
+ metadata=metadata["metadata"],
+ )
+ # Download the content to make sure it's "working as expected".
+ await check_content(blob_name, actual_contents, authorized_transport)
+ # Make sure the upload is tombstoned.
+ await check_tombstoned(
+ upload, authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE
+ )
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+@pytest.mark.asyncio
+async def test_multipart_upload_with_bad_checksum(
+ authorized_transport, checksum, bucket
+):
+ with open(ICO_FILE, "rb") as file_obj:
+ actual_contents = file_obj.read()
+
+ blob_name = os.path.basename(ICO_FILE)
+ await check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.MULTIPART_UPLOAD
+ upload = resumable_requests.MultipartUpload(upload_url, checksum=checksum)
+ # Transmit the resource.
+ metadata = {"name": blob_name, "metadata": {"color": "yellow"}}
+ fake_checksum_object = _helpers._get_checksum_object(checksum)
+ fake_checksum_object.update(b"bad data")
+ fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
+ fake_checksum_object.digest()
+ )
+ with mock.patch.object(
+ _helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest
+ ):
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await upload.transmit(
+ authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE
+ )
+ response = exc_info.value.response
+ message = await response.text()
+ # Attempt to verify that this is a checksum mismatch error.
+ assert checksum.upper() in message
+ assert fake_prepared_checksum_digest in message
+
+ # Make sure the upload is tombstoned.
+ await check_tombstoned(
+ upload, authorized_transport, actual_contents, metadata, ICO_CONTENT_TYPE
+ )
+
+
+@pytest.mark.asyncio
+async def test_multipart_upload_with_headers(authorized_transport, bucket, cleanup):
+ blob_name = "some-multipart-stuff.bin"
+ # Make sure to clean up the uploaded blob when we are done.
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+
+ # Create the actual upload object.
+ upload_url = utils.MULTIPART_UPLOAD
+ headers = utils.get_encryption_headers()
+ upload = resumable_requests.MultipartUpload(upload_url, headers=headers)
+ # Transmit the resource.
+ metadata = {"name": blob_name}
+ data = b"Other binary contents\x03\x04\x05."
+ response = await upload.transmit(
+ authorized_transport, data, metadata, BYTES_CONTENT_TYPE
+ )
+ await check_response(
+ response, blob_name, actual_contents=data, content_type=BYTES_CONTENT_TYPE
+ )
+ # Download the content to make sure it's "working as expected".
+ await check_content(blob_name, data, authorized_transport, headers=headers)
+ # Make sure the upload is tombstoned.
+ await check_tombstoned(
+ upload, authorized_transport, data, metadata, BYTES_CONTENT_TYPE
+ )
+
+
+async def _resumable_upload_helper(
+ authorized_transport, stream, cleanup, checksum=None, headers=None
+):
+ blob_name = os.path.basename(stream.name)
+ # Make sure to clean up the uploaded blob when we are done.
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+ # Create the actual upload object.
+ chunk_size = _async_resumable_media.UPLOAD_CHUNK_SIZE
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, chunk_size, headers=headers, checksum=checksum
+ )
+ # Initiate the upload.
+ metadata = {"name": blob_name, "metadata": {"direction": "north"}}
+ response = await upload.initiate(
+ authorized_transport, stream, metadata, JPEG_CONTENT_TYPE
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ await check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Actually upload the file in chunks.
+ num_chunks = await transmit_chunks(
+ upload, authorized_transport, blob_name, metadata["metadata"]
+ )
+ assert num_chunks == get_num_chunks(upload.total_bytes, chunk_size)
+ # Download the content to make sure it's "working as expected".
+ stream.seek(0)
+ actual_contents = stream.read()
+ await check_content(
+ blob_name, actual_contents, authorized_transport, headers=headers
+ )
+ # Make sure the upload is tombstoned.
+ await check_tombstoned(upload, authorized_transport)
+
+
+@pytest.mark.asyncio
+async def test_resumable_upload(authorized_transport, img_stream, bucket, cleanup):
+ await _resumable_upload_helper(authorized_transport, img_stream, cleanup)
+
+
+@pytest.mark.asyncio
+async def test_resumable_upload_with_headers(
+ authorized_transport, img_stream, bucket, cleanup
+):
+ headers = utils.get_encryption_headers()
+ await _resumable_upload_helper(
+ authorized_transport, img_stream, cleanup, headers=headers
+ )
+
+
+@pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+@pytest.mark.asyncio
+async def test_resumable_upload_with_bad_checksum(
+ authorized_transport, img_stream, bucket, cleanup, checksum
+):
+ fake_checksum_object = _helpers._get_checksum_object(checksum)
+ fake_checksum_object.update(b"bad data")
+ fake_prepared_checksum_digest = _helpers.prepare_checksum_digest(
+ fake_checksum_object.digest()
+ )
+ with mock.patch.object(
+ _helpers, "prepare_checksum_digest", return_value=fake_prepared_checksum_digest
+ ):
+ with pytest.raises(common.DataCorruption) as exc_info:
+ await _resumable_upload_helper(
+ authorized_transport, img_stream, cleanup, checksum=checksum
+ )
+ expected_checksums = {"md5": "1bsd83IYNug8hd+V1ING3Q==", "crc32c": "YQGPxA=="}
+ expected_message = (
+ _async_resumable_media._upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
+ checksum.upper(),
+ fake_prepared_checksum_digest,
+ expected_checksums[checksum],
+ )
+ )
+ assert exc_info.value.args[0] == expected_message
+
+
+@pytest.mark.asyncio
+async def test_resumable_upload_bad_chunk_size(authorized_transport, img_stream):
+ blob_name = os.path.basename(img_stream.name)
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, _async_resumable_media.UPLOAD_CHUNK_SIZE
+ )
+ # Modify the ``upload`` **after** construction so we can
+ # use a bad chunk size.
+ upload._chunk_size = 1024
+ assert upload._chunk_size < _async_resumable_media.UPLOAD_CHUNK_SIZE
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ response = await upload.initiate(
+ authorized_transport, img_stream, metadata, JPEG_CONTENT_TYPE
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ await check_initiate(response, upload, img_stream, authorized_transport, metadata)
+ # Make the first request and verify that it fails.
+ await check_bad_chunk(upload, authorized_transport)
+ # Reset the chunk size (and the stream) and verify the "resumable"
+ # URL is unusable.
+ upload._chunk_size = _async_resumable_media.UPLOAD_CHUNK_SIZE
+ img_stream.seek(0)
+ upload._invalid = False
+ await check_bad_chunk(upload, authorized_transport)
+
+
+async def sabotage_and_recover(upload, stream, transport, chunk_size):
+ assert upload.bytes_uploaded == chunk_size
+ assert stream.tell() == chunk_size
+ # "Fake" that the instance is in an invalid state.
+ upload._invalid = True
+ stream.seek(0) # Seek to the wrong place.
+ upload._bytes_uploaded = 0 # Make ``bytes_uploaded`` wrong as well.
+ # Recover the (artifically) invalid upload.
+ response = await upload.recover(transport)
+ assert response.status == http.client.PERMANENT_REDIRECT
+ assert not upload.invalid
+ assert upload.bytes_uploaded == chunk_size
+ assert stream.tell() == chunk_size
+
+
+async def _resumable_upload_recover_helper(authorized_transport, cleanup, headers=None):
+ blob_name = "some-bytes.bin"
+ chunk_size = _async_resumable_media.UPLOAD_CHUNK_SIZE
+ data = b"123" * chunk_size # 3 chunks worth.
+ # Make sure to clean up the uploaded blob when we are done.
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(
+ utils.RESUMABLE_UPLOAD, chunk_size, headers=headers
+ )
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ stream = io.BytesIO(data)
+ response = await upload.initiate(
+ authorized_transport, stream, metadata, BYTES_CONTENT_TYPE
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ await check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Make the first request.
+ response = await upload.transmit_next_chunk(authorized_transport)
+ assert response.status == http.client.PERMANENT_REDIRECT
+ # Call upload.recover().
+ await sabotage_and_recover(upload, stream, authorized_transport, chunk_size)
+ # Now stream what remains.
+ num_chunks = await transmit_chunks(
+ upload,
+ authorized_transport,
+ blob_name,
+ None,
+ num_chunks=1,
+ content_type=BYTES_CONTENT_TYPE,
+ )
+ assert num_chunks == 3
+ # Download the content to make sure it's "working as expected".
+ actual_contents = stream.getvalue()
+ await check_content(
+ blob_name, actual_contents, authorized_transport, headers=headers
+ )
+ # Make sure the upload is tombstoned.
+ await check_tombstoned(upload, authorized_transport)
+
+
+@pytest.mark.asyncio
+async def test_resumable_upload_recover(authorized_transport, bucket, cleanup):
+ await _resumable_upload_recover_helper(authorized_transport, cleanup)
+
+
+@pytest.mark.asyncio
+async def test_resumable_upload_recover_with_headers(
+ authorized_transport, bucket, cleanup
+):
+ headers = utils.get_encryption_headers()
+ await _resumable_upload_recover_helper(
+ authorized_transport, cleanup, headers=headers
+ )
+
+
+class TestResumableUploadUnknownSize(object):
+ @staticmethod
+ def _check_range_sent(response, start, end, total):
+ headers_sent = response.request_info.headers
+ if start is None and end is None:
+ expected_content_range = "bytes */{:d}".format(total)
+ else:
+ # Allow total to be an int or a string "*"
+ expected_content_range = "bytes {:d}-{:d}/{}".format(start, end, total)
+
+ assert headers_sent["content-range"] == expected_content_range
+
+ @staticmethod
+ def _check_range_received(response, size):
+ assert response.headers["range"] == "bytes=0-{:d}".format(size - 1)
+
+ async def _check_partial(self, upload, response, chunk_size, num_chunks):
+ start_byte = (num_chunks - 1) * chunk_size
+ end_byte = num_chunks * chunk_size - 1
+
+ assert not upload.finished
+ assert upload.bytes_uploaded == end_byte + 1
+ assert response.status == http.client.PERMANENT_REDIRECT
+ content = await response.content.read()
+ assert content == b""
+
+ self._check_range_sent(response, start_byte, end_byte, "*")
+ self._check_range_received(response, end_byte + 1)
+
+ @pytest.mark.asyncio
+ async def test_smaller_than_chunk_size(self, authorized_transport, bucket, cleanup):
+ blob_name = os.path.basename(ICO_FILE)
+ chunk_size = _async_resumable_media.UPLOAD_CHUNK_SIZE
+ # Make sure to clean up the uploaded blob when we are done.
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+ # Make sure the blob is smaller than the chunk size.
+ total_bytes = os.path.getsize(ICO_FILE)
+ assert total_bytes < chunk_size
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(utils.RESUMABLE_UPLOAD, chunk_size)
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ with open(ICO_FILE, "rb") as stream:
+ response = await upload.initiate(
+ authorized_transport,
+ stream,
+ metadata,
+ ICO_CONTENT_TYPE,
+ stream_final=False,
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ await check_initiate(
+ response, upload, stream, authorized_transport, metadata
+ )
+ # Make sure total bytes was never set.
+ assert upload.total_bytes is None
+ # Make the **ONLY** request.
+ response = await upload.transmit_next_chunk(authorized_transport)
+ self._check_range_sent(response, 0, total_bytes - 1, total_bytes)
+ await check_response(response, blob_name, total_bytes=total_bytes)
+ # Download the content to make sure it's "working as expected".
+ stream.seek(0)
+ actual_contents = stream.read()
+ await check_content(blob_name, actual_contents, authorized_transport)
+ # Make sure the upload is tombstoned.
+ await check_tombstoned(upload, authorized_transport)
+
+ @pytest.mark.asyncio
+ async def test_finish_at_chunk(self, authorized_transport, bucket, cleanup):
+ blob_name = "some-clean-stuff.bin"
+ chunk_size = _async_resumable_media.UPLOAD_CHUNK_SIZE
+ # Make sure to clean up the uploaded blob when we are done.
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+ # Make sure the blob size is an exact multiple of the chunk size.
+ data = b"ab" * chunk_size
+ total_bytes = len(data)
+ stream = io.BytesIO(data)
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(utils.RESUMABLE_UPLOAD, chunk_size)
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ response = await upload.initiate(
+ authorized_transport,
+ stream,
+ metadata,
+ BYTES_CONTENT_TYPE,
+ stream_final=False,
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ await check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Make sure total bytes was never set.
+ assert upload.total_bytes is None
+ # Make three requests.
+ response0 = await upload.transmit_next_chunk(authorized_transport)
+ await self._check_partial(upload, response0, chunk_size, 1)
+
+ response1 = await upload.transmit_next_chunk(authorized_transport)
+ await self._check_partial(upload, response1, chunk_size, 2)
+
+ response2 = await upload.transmit_next_chunk(authorized_transport)
+ assert upload.finished
+ # Verify the "clean-up" request.
+ assert upload.bytes_uploaded == 2 * chunk_size
+ await check_response(
+ response2,
+ blob_name,
+ actual_contents=data,
+ total_bytes=total_bytes,
+ content_type=BYTES_CONTENT_TYPE,
+ )
+ self._check_range_sent(response2, None, None, 2 * chunk_size)
+
+ @staticmethod
+ def _add_bytes(stream, data):
+ curr_pos = stream.tell()
+ stream.write(data)
+ # Go back to where we were before the write.
+ stream.seek(curr_pos)
+
+ @pytest.mark.asyncio
+ async def test_interleave_writes(self, authorized_transport, bucket, cleanup):
+ blob_name = "some-moar-stuff.bin"
+ chunk_size = _async_resumable_media.UPLOAD_CHUNK_SIZE
+ # Make sure to clean up the uploaded blob when we are done.
+ await cleanup(blob_name, authorized_transport)
+ await check_does_not_exist(authorized_transport, blob_name)
+ # Start out the blob as a single chunk (but we will add to it).
+ stream = io.BytesIO(b"Z" * chunk_size)
+ # Create the actual upload object.
+ upload = resumable_requests.ResumableUpload(utils.RESUMABLE_UPLOAD, chunk_size)
+ # Initiate the upload.
+ metadata = {"name": blob_name}
+ response = await upload.initiate(
+ authorized_transport,
+ stream,
+ metadata,
+ BYTES_CONTENT_TYPE,
+ stream_final=False,
+ )
+ # Make sure ``initiate`` succeeded and did not mangle the stream.
+ await check_initiate(response, upload, stream, authorized_transport, metadata)
+ # Make sure total bytes was never set.
+ assert upload.total_bytes is None
+ # Make three requests.
+ response0 = await upload.transmit_next_chunk(authorized_transport)
+ await self._check_partial(upload, response0, chunk_size, 1)
+ # Add another chunk before sending.
+ self._add_bytes(stream, b"K" * chunk_size)
+ response1 = await upload.transmit_next_chunk(authorized_transport)
+ await self._check_partial(upload, response1, chunk_size, 2)
+ # Add more bytes, but make sure less than a full chunk.
+ last_chunk = 155
+ self._add_bytes(stream, b"r" * last_chunk)
+ response2 = await upload.transmit_next_chunk(authorized_transport)
+ assert upload.finished
+ # Verify the "clean-up" request.
+ total_bytes = 2 * chunk_size + last_chunk
+ assert upload.bytes_uploaded == total_bytes
+ await check_response(
+ response2,
+ blob_name,
+ actual_contents=stream.getvalue(),
+ total_bytes=total_bytes,
+ content_type=BYTES_CONTENT_TYPE,
+ )
+ self._check_range_sent(response2, 2 * chunk_size, total_bytes - 1, total_bytes)
diff --git a/packages/google-resumable-media/tests_async/system/utils.py b/packages/google-resumable-media/tests_async/system/utils.py
new file mode 100644
index 000000000000..620b2c99c1d3
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/system/utils.py
@@ -0,0 +1,69 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import base64
+import hashlib
+import time
+
+
+BUCKET_NAME = "grpm-systest-{}".format(int(1000 * time.time()))
+BUCKET_POST_URL = "https://www.googleapis.com/storage/v1/b/"
+BUCKET_URL = "https://www.googleapis.com/storage/v1/b/{}".format(BUCKET_NAME)
+
+_DOWNLOAD_BASE = "https://www.googleapis.com/download/storage/v1/b/{}".format(
+ BUCKET_NAME
+)
+DOWNLOAD_URL_TEMPLATE = _DOWNLOAD_BASE + "/o/{blob_name}?alt=media"
+
+_UPLOAD_BASE = (
+ "https://www.googleapis.com/upload/storage/v1/b/{}".format(BUCKET_NAME)
+ + "/o?uploadType="
+)
+SIMPLE_UPLOAD_TEMPLATE = _UPLOAD_BASE + "media&name={blob_name}"
+MULTIPART_UPLOAD = _UPLOAD_BASE + "multipart"
+RESUMABLE_UPLOAD = _UPLOAD_BASE + "resumable"
+
+METADATA_URL_TEMPLATE = BUCKET_URL + "/o/{blob_name}"
+
+GCS_RW_SCOPE = "https://www.googleapis.com/auth/devstorage.read_write"
+# Generated using random.choice() with all 256 byte choices.
+ENCRYPTION_KEY = (
+ b"R\xb8\x1b\x94T\xea_\xa8\x93\xae\xd1\xf6\xfca\x15\x0ekA"
+ b"\x08 Y\x13\xe2\n\x02i\xadc\xe2\xd99x"
+)
+
+
+def get_encryption_headers(key=ENCRYPTION_KEY):
+ """Builds customer-supplied encryption key headers
+
+ See `Managing Data Encryption`_ for more details.
+
+ Args:
+ key (bytes): 32 byte key to build request key and hash.
+
+ Returns:
+ Dict[str, str]: The algorithm, key and key-SHA256 headers.
+
+ .. _Managing Data Encryption:
+ https://cloud.google.com/storage/docs/encryption
+ """
+ key_hash = hashlib.sha256(key).digest()
+ key_hash_b64 = base64.b64encode(key_hash)
+ key_b64 = base64.b64encode(key)
+
+ return {
+ "x-goog-encryption-algorithm": "AES256",
+ "x-goog-encryption-key": key_b64.decode("utf-8"),
+ "x-goog-encryption-key-sha256": key_hash_b64.decode("utf-8"),
+ }
diff --git a/packages/google-resumable-media/tests_async/unit/__init__.py b/packages/google-resumable-media/tests_async/unit/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/unit/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests_async/unit/requests/__init__.py b/packages/google-resumable-media/tests_async/unit/requests/__init__.py
new file mode 100644
index 000000000000..7c07b241f066
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/unit/requests/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/packages/google-resumable-media/tests_async/unit/requests/test__helpers.py b/packages/google-resumable-media/tests_async/unit/requests/test__helpers.py
new file mode 100644
index 000000000000..333c2dec99c7
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/unit/requests/test__helpers.py
@@ -0,0 +1,123 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+
+import aiohttp # type: ignore
+import mock
+import pytest # type: ignore
+
+from google._async_resumable_media.requests import _request_helpers as _helpers
+
+# async version takes a single timeout, not a tuple of connect, read timeouts.
+EXPECTED_TIMEOUT = aiohttp.ClientTimeout(connect=61, sock_read=60)
+
+
+class TestRequestsMixin(object):
+ def test__get_status_code(self):
+ status_code = int(http.client.OK)
+ response = _make_response(status_code)
+ assert status_code == _helpers.RequestsMixin._get_status_code(response)
+
+ def test__get_headers(self):
+ headers = {"fruit": "apple"}
+ response = mock.Mock(
+ _headers=headers, headers=headers, spec=["_headers", "headers"]
+ )
+ assert headers == _helpers.RequestsMixin._get_headers(response)
+
+ @pytest.mark.asyncio
+ async def test__get_body(self):
+ body = b"This is the payload."
+ content_stream = mock.AsyncMock(spec=["__call__", "read"])
+ content_stream.read = mock.AsyncMock(spec=["__call__"], return_value=body)
+ response = mock.AsyncMock(
+ content=content_stream,
+ spec=["__call__", "content"],
+ )
+ temp = await _helpers.RequestsMixin._get_body(response)
+ assert body == temp
+
+
+class TestRawRequestsMixin(object):
+ class AsyncByteStream:
+ def __init__(self, bytes):
+ self._byte_stream = io.BytesIO(bytes)
+
+ async def read(self):
+ return self._byte_stream.read()
+
+ @pytest.mark.asyncio
+ async def test__get_body(self):
+ body = b"This is the payload."
+ response = mock.Mock(
+ content=TestRawRequestsMixin.AsyncByteStream(body), spec=["content"]
+ )
+ assert body == await _helpers.RawRequestsMixin._get_body(response)
+
+
+@pytest.mark.asyncio
+async def test_http_request():
+ transport, response = _make_transport(http.client.OK)
+ method = "POST"
+ url = "http://test.invalid"
+ data = mock.sentinel.data
+ headers = {"one": "fish", "blue": "fish"}
+ timeout = mock.sentinel.timeout
+ ret_val = await _helpers.http_request(
+ transport,
+ method,
+ url,
+ data=data,
+ headers=headers,
+ extra1=b"work",
+ extra2=125.5,
+ timeout=timeout,
+ )
+
+ assert ret_val is response
+ transport.request.assert_called_once_with(
+ method,
+ url,
+ data=data,
+ headers=headers,
+ extra1=b"work",
+ extra2=125.5,
+ timeout=timeout,
+ )
+
+
+@pytest.mark.asyncio
+async def test_http_request_defaults():
+ transport, response = _make_transport(http.client.OK)
+ method = "POST"
+ url = "http://test.invalid"
+
+ ret_val = await _helpers.http_request(transport, method, url)
+ assert ret_val is response
+ transport.request.assert_called_once_with(
+ method, url, data=None, headers=None, timeout=EXPECTED_TIMEOUT
+ )
+
+
+def _make_response(status_code):
+ return mock.AsyncMock(status=status_code, spec=["status"])
+
+
+def _make_transport(status_code):
+ response = _make_response(status_code)
+ transport = mock.AsyncMock(spec=["request"])
+ transport.request = mock.AsyncMock(spec=["__call__"], return_value=response)
+ return transport, response
diff --git a/packages/google-resumable-media/tests_async/unit/requests/test_download.py b/packages/google-resumable-media/tests_async/unit/requests/test_download.py
new file mode 100644
index 000000000000..16c23a49327a
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/unit/requests/test_download.py
@@ -0,0 +1,857 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+
+import aiohttp # type: ignore
+import mock
+import pytest # type: ignore
+
+
+from google.resumable_media import common
+from google._async_resumable_media import _helpers
+from google._async_resumable_media.requests import download as download_mod
+from tests.unit.requests import test_download as sync_test
+
+EXPECTED_TIMEOUT = aiohttp.ClientTimeout(
+ total=None, connect=61, sock_read=60, sock_connect=None
+)
+
+
+class TestDownload(object):
+ @pytest.mark.asyncio
+ async def test__write_to_stream_no_hash_check(self):
+ stream = io.BytesIO()
+ download = download_mod.Download(sync_test.EXAMPLE_URL, stream=stream)
+
+ chunk1 = b"right now, "
+ chunk2 = b"but a little later"
+ response = _mock_response(chunks=[chunk1, chunk2], headers={})
+
+ ret_val = await download._write_to_stream(response)
+ assert ret_val is None
+
+ assert stream.getvalue() == chunk1 + chunk2
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ @pytest.mark.asyncio
+ async def test__write_to_stream_with_hash_check_success(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.Download(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ header_value = "crc32c=qmNCyg==,md5=fPAJHnnoi/+NadyNxT2c2w=="
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ ret_val = await download._write_to_stream(response)
+ assert ret_val is None
+
+ assert stream.getvalue() == chunk1 + chunk2 + chunk3
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test__write_to_stream_with_hash_check_fail(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.Download(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ with pytest.raises(common.DataCorruption) as exc_info:
+ await download._write_to_stream(response)
+
+ assert not download.finished
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 1
+ if checksum == "md5":
+ good_checksum = "fPAJHnnoi/+NadyNxT2c2w=="
+ else:
+ good_checksum = "qmNCyg=="
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ sync_test.EXAMPLE_URL,
+ bad_checksum,
+ good_checksum,
+ checksum_type=checksum.upper(),
+ )
+ assert msg in error.args[0]
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ async def test__write_to_stream_no_checksum_validation_for_partial_response(
+ self, checksum
+ ):
+ stream = io.BytesIO()
+ download = download_mod.Download(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunk1 = b"first chunk"
+ response = _mock_response(status=http.client.PARTIAL_CONTENT, chunks=[chunk1])
+
+ # Make sure that the checksum is not validated.
+ with mock.patch(
+ "google.resumable_media._helpers.prepare_checksum_digest",
+ return_value=None,
+ ) as prepare_checksum_digest:
+ await download._write_to_stream(response)
+ assert not prepare_checksum_digest.called
+
+ assert not download.finished
+
+ @pytest.mark.asyncio
+ async def test__write_to_stream_with_invalid_checksum_type(self):
+ BAD_CHECKSUM_TYPE = "badsum"
+
+ stream = io.BytesIO()
+ download = download_mod.Download(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=BAD_CHECKSUM_TYPE
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ with pytest.raises(ValueError) as exc_info:
+ await download._write_to_stream(response)
+
+ assert not download.finished
+
+ error = exc_info.value
+ assert error.args[0] == "checksum must be ``'md5'``, ``'crc32c'`` or ``None``"
+
+ @pytest.mark.asyncio
+ async def _consume_helper(
+ self,
+ stream=None,
+ end=65536,
+ headers=None,
+ chunks=(),
+ response_headers=None,
+ checksum="md5",
+ timeout=None,
+ ):
+ download = download_mod.Download(
+ sync_test.EXAMPLE_URL, stream=stream, end=end, headers=headers
+ )
+ transport = mock.AsyncMock(spec=["request"])
+ mockResponse = _mock_response(chunks=chunks, headers=response_headers)
+ transport.request = mock.AsyncMock(spec=["__call__"], return_value=mockResponse)
+
+ assert not download.finished
+
+ if timeout is not None:
+ ret_val = await download.consume(transport, timeout=timeout)
+ else:
+ ret_val = await download.consume(transport)
+
+ assert ret_val is transport.request.return_value
+
+ called_kwargs = {
+ "data": None,
+ "headers": download._headers,
+ "timeout": EXPECTED_TIMEOUT if timeout is None else timeout,
+ }
+
+ if chunks:
+ assert stream is not None
+ called_kwargs["stream"] = True
+ transport.request.assert_called_once_with(
+ "GET", sync_test.EXAMPLE_URL, **called_kwargs
+ )
+
+ range_bytes = "bytes={:d}-{:d}".format(0, end)
+ assert download._headers["range"] == range_bytes
+ assert download.finished
+
+ return transport
+
+ @pytest.mark.asyncio
+ async def test_consume(self):
+ await self._consume_helper()
+
+ @pytest.mark.asyncio
+ async def test_consume_with_custom_timeout(self):
+ await self._consume_helper(timeout=14.7)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ @pytest.mark.asyncio
+ async def test_consume_with_stream(self, checksum):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ # transport = await self._consume_helper(stream=stream, chunks=chunks, checksum=checksum)
+ await self._consume_helper(stream=stream, chunks=chunks, checksum=checksum)
+
+ assert stream.getvalue() == b"".join(chunks)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test_consume_with_stream_hash_check_success(self, checksum):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ header_value = "crc32c=UNIQxg==,md5=JvS1wjMvfbCXgEGeaJJLDQ=="
+ headers = {_helpers._HASH_HEADER: header_value}
+ await self._consume_helper(
+ stream=stream, chunks=chunks, response_headers=headers, checksum=checksum
+ )
+
+ assert stream.getvalue() == b"".join(chunks)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test_consume_with_stream_hash_check_fail(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.Download(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunks = (b"zero zero", b"niner tango")
+ bad_checksum = "anVzdCBub3QgdGhpcyAxLA=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+
+ transport = mock.AsyncMock(spec=["request"])
+ mockResponse = _mock_response(chunks=chunks, headers=headers)
+ transport.request = mock.AsyncMock(spec=["__call__"], return_value=mockResponse)
+
+ assert not download.finished
+ with pytest.raises(common.DataCorruption) as exc_info:
+ await download.consume(transport)
+
+ assert stream.getvalue() == b"".join(chunks)
+ assert download.finished
+ assert download._headers == {}
+
+ error = exc_info.value
+ assert error.response is transport.request.return_value
+ assert len(error.args) == 1
+ if checksum == "md5":
+ good_checksum = "1A/dxEpys717C6FH7FIWDw=="
+ else:
+ good_checksum = "GvNZlg=="
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ sync_test.EXAMPLE_URL,
+ bad_checksum,
+ good_checksum,
+ checksum_type=checksum.upper(),
+ )
+ assert msg in error.args[0]
+
+ # Check mocks.
+ transport.request.assert_called_once_with(
+ "GET",
+ sync_test.EXAMPLE_URL,
+ data=None,
+ headers={},
+ stream=True,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ @pytest.mark.asyncio
+ async def test_consume_with_headers(self):
+ headers = {} # Empty headers
+ end = 16383
+ await self._consume_helper(end=end, headers=headers)
+ range_bytes = "bytes={:d}-{:d}".format(0, end)
+ # Make sure the headers have been modified.
+ assert headers == {"range": range_bytes}
+
+
+class TestRawDownload(object):
+ @pytest.mark.asyncio
+ async def test__write_to_stream_no_hash_check(self):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(sync_test.EXAMPLE_URL, stream=stream)
+
+ chunk1 = b"right now, "
+ chunk2 = b"but a little later"
+ response = _mock_raw_response(chunks=[chunk1, chunk2], headers={})
+ ret_val = await download._write_to_stream(response)
+ assert ret_val is None
+
+ assert stream.getvalue() == chunk1 + chunk2
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test__write_to_stream_with_hash_check_success(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ header_value = "crc32c=qmNCyg==,md5=fPAJHnnoi/+NadyNxT2c2w=="
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ ret_val = await download._write_to_stream(response)
+ assert ret_val is None
+
+ assert stream.getvalue() == chunk1 + chunk2 + chunk3
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test__write_to_stream_with_hash_check_fail(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_raw_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ with pytest.raises(common.DataCorruption) as exc_info:
+ await download._write_to_stream(response)
+
+ assert not download.finished
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 1
+ if checksum == "md5":
+ good_checksum = "fPAJHnnoi/+NadyNxT2c2w=="
+ else:
+ good_checksum = "qmNCyg=="
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ sync_test.EXAMPLE_URL,
+ bad_checksum,
+ good_checksum,
+ checksum_type=checksum.upper(),
+ )
+ assert msg in error.args[0]
+
+ @pytest.mark.asyncio
+ async def test__write_to_stream_with_invalid_checksum_type(self):
+ BAD_CHECKSUM_TYPE = "badsum"
+
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=BAD_CHECKSUM_TYPE
+ )
+
+ chunk1 = b"first chunk, count starting at 0. "
+ chunk2 = b"second chunk, or chunk 1, which is better? "
+ chunk3 = b"ordinals and numerals and stuff."
+ bad_checksum = "d3JvbmcgbiBtYWRlIHVwIQ=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ response = _mock_response(chunks=[chunk1, chunk2, chunk3], headers=headers)
+
+ with pytest.raises(ValueError) as exc_info:
+ await download._write_to_stream(response)
+
+ assert not download.finished
+
+ error = exc_info.value
+ assert error.args[0] == "checksum must be ``'md5'``, ``'crc32c'`` or ``None``"
+
+ async def _consume_helper(
+ self,
+ stream=None,
+ end=65536,
+ headers=None,
+ chunks=(),
+ response_headers=None,
+ checksum=None,
+ timeout=None,
+ ):
+ download = download_mod.RawDownload(
+ sync_test.EXAMPLE_URL, stream=stream, end=end, headers=headers
+ )
+
+ transport = mock.AsyncMock(spec=["request"])
+ mockResponse = _mock_raw_response(chunks=chunks, headers=response_headers)
+ transport.request = mock.AsyncMock(spec=["__call__"], return_value=mockResponse)
+
+ assert not download.finished
+ ret_val = await download.consume(transport)
+ assert ret_val is transport.request.return_value
+
+ if chunks:
+ assert stream is not None
+ transport.request.assert_called_once_with(
+ "GET",
+ sync_test.EXAMPLE_URL,
+ data=None,
+ headers=download._headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ range_bytes = "bytes={:d}-{:d}".format(0, end)
+ assert download._headers["range"] == range_bytes
+ assert download.finished
+
+ return transport
+
+ @pytest.mark.asyncio
+ async def test_consume(self):
+ await self._consume_helper()
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ @pytest.mark.asyncio
+ async def test_consume_with_stream(self, checksum):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ await self._consume_helper(stream=stream, chunks=chunks, checksum=checksum)
+
+ assert stream.getvalue() == b"".join(chunks)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c", None])
+ @pytest.mark.asyncio
+ async def test_consume_with_stream_hash_check_success(self, checksum):
+ stream = io.BytesIO()
+ chunks = (b"up down ", b"charlie ", b"brown")
+ header_value = "crc32c=UNIQxg==,md5=JvS1wjMvfbCXgEGeaJJLDQ=="
+ headers = {_helpers._HASH_HEADER: header_value}
+
+ await self._consume_helper(
+ stream=stream, chunks=chunks, response_headers=headers, checksum=checksum
+ )
+
+ assert stream.getvalue() == b"".join(chunks)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test_consume_with_stream_hash_check_fail(self, checksum):
+ stream = io.BytesIO()
+ download = download_mod.RawDownload(
+ sync_test.EXAMPLE_URL, stream=stream, checksum=checksum
+ )
+
+ chunks = (b"zero zero", b"niner tango")
+ bad_checksum = "anVzdCBub3QgdGhpcyAxLA=="
+ header_value = "crc32c={bad},md5={bad}".format(bad=bad_checksum)
+ headers = {_helpers._HASH_HEADER: header_value}
+ transport = mock.AsyncMock(spec=["request"])
+ mockResponse = _mock_raw_response(chunks=chunks, headers=headers)
+ transport.request = mock.AsyncMock(spec=["__call__"], return_value=mockResponse)
+
+ assert not download.finished
+ with pytest.raises(common.DataCorruption) as exc_info:
+ await download.consume(transport)
+
+ assert stream.getvalue() == b"".join(chunks)
+ assert download.finished
+ assert download._headers == {}
+
+ error = exc_info.value
+ assert error.response is transport.request.return_value
+ assert len(error.args) == 1
+ if checksum == "md5":
+ good_checksum = "1A/dxEpys717C6FH7FIWDw=="
+ else:
+ good_checksum = "GvNZlg=="
+ msg = download_mod._CHECKSUM_MISMATCH.format(
+ sync_test.EXAMPLE_URL,
+ bad_checksum,
+ good_checksum,
+ checksum_type=checksum.upper(),
+ )
+ assert msg in error.args[0]
+
+ # Check mocks.
+ transport.request.assert_called_once_with(
+ "GET",
+ sync_test.EXAMPLE_URL,
+ data=None,
+ headers={},
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ @pytest.mark.asyncio
+ async def test_consume_with_headers(self):
+ headers = {} # Empty headers
+ end = 16383
+ await self._consume_helper(end=end, headers=headers)
+ range_bytes = "bytes={:d}-{:d}".format(0, end)
+ # Make sure the headers have been modified.
+ assert headers == {"range": range_bytes}
+
+
+class TestChunkedDownload(object):
+ @staticmethod
+ def _response_content_range(start_byte, end_byte, total_bytes):
+ return "bytes {:d}-{:d}/{:d}".format(start_byte, end_byte, total_bytes)
+
+ def _response_headers(self, start_byte, end_byte, total_bytes):
+ content_length = end_byte - start_byte + 1
+ resp_range = self._response_content_range(start_byte, end_byte, total_bytes)
+ return {
+ "content-length": "{:d}".format(content_length),
+ "content-range": resp_range,
+ }
+
+ def _mock_response(
+ self, start_byte, end_byte, total_bytes, content=None, status_code=None
+ ):
+ response_headers = self._response_headers(start_byte, end_byte, total_bytes)
+ content_stream = mock.AsyncMock(spec=["__call__", "read"])
+ content_stream.read = mock.AsyncMock(spec=["__call__"], return_value=content)
+ return mock.AsyncMock(
+ content=content_stream,
+ _headers=response_headers,
+ headers=response_headers,
+ status=status_code,
+ spec=["content", "headers", "status"],
+ )
+
+ @pytest.mark.asyncio
+ async def test_consume_next_chunk_already_finished(self):
+ download = download_mod.ChunkedDownload(sync_test.EXAMPLE_URL, 512, None)
+ download._finished = True
+ with pytest.raises(ValueError):
+ await download.consume_next_chunk(None)
+
+ def _mock_transport(self, start, chunk_size, total_bytes, content=b""):
+ transport = mock.AsyncMock(spec=["request"])
+ assert len(content) == chunk_size
+ mockResponse = self._mock_response(
+ start,
+ start + chunk_size - 1,
+ total_bytes,
+ content=content,
+ status_code=int(http.client.OK),
+ )
+ transport.request = mock.AsyncMock(spec=["__call__"], return_value=mockResponse)
+
+ return transport
+
+ @pytest.mark.asyncio
+ async def test_consume_next_chunk(self):
+ start = 1536
+ stream = io.BytesIO()
+ data = b"Just one chunk."
+ chunk_size = len(data)
+ download = download_mod.ChunkedDownload(
+ sync_test.EXAMPLE_URL, chunk_size, stream, start=start
+ )
+ total_bytes = 16384
+ transport = self._mock_transport(start, chunk_size, total_bytes, content=data)
+
+ # Verify the internal state before consuming a chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually consume the chunk and check the output.
+ ret_val = await download.consume_next_chunk(transport)
+ assert ret_val is transport.request.return_value
+ range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1)
+ download_headers = {"range": range_bytes}
+ transport.request.assert_called_once_with(
+ "GET",
+ sync_test.EXAMPLE_URL,
+ data=None,
+ headers=download_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+ assert stream.getvalue() == data
+ # Go back and check the internal state after consuming the chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == chunk_size
+ assert download.total_bytes == total_bytes
+
+ @pytest.mark.asyncio
+ async def test_consume_next_chunk_with_custom_timeout(self):
+ start = 1536
+ stream = io.BytesIO()
+ data = b"Just one chunk."
+ chunk_size = len(data)
+ download = download_mod.ChunkedDownload(
+ sync_test.EXAMPLE_URL, chunk_size, stream, start=start
+ )
+ total_bytes = 16384
+ transport = self._mock_transport(start, chunk_size, total_bytes, content=data)
+
+ # Actually consume the chunk and check the output.
+ await download.consume_next_chunk(transport, timeout=14.7)
+
+ range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1)
+ download_headers = {"range": range_bytes}
+ transport.request.assert_called_once_with(
+ "GET",
+ sync_test.EXAMPLE_URL,
+ data=None,
+ headers=download_headers,
+ timeout=14.7,
+ )
+
+
+class TestRawChunkedDownload(object):
+ @staticmethod
+ def _response_content_range(start_byte, end_byte, total_bytes):
+ return "bytes {:d}-{:d}/{:d}".format(start_byte, end_byte, total_bytes)
+
+ def _response_headers(self, start_byte, end_byte, total_bytes):
+ content_length = end_byte - start_byte + 1
+ resp_range = self._response_content_range(start_byte, end_byte, total_bytes)
+ return {
+ "content-length": "{:d}".format(content_length),
+ "content-range": resp_range,
+ }
+
+ def _mock_response(
+ self, start_byte, end_byte, total_bytes, content=None, status_code=None
+ ):
+ response_headers = self._response_headers(start_byte, end_byte, total_bytes)
+ content_stream = mock.AsyncMock(spec=["__call__", "read"])
+ content_stream.read = mock.AsyncMock(spec=["__call__"], return_value=content)
+ return mock.AsyncMock(
+ content=content_stream,
+ _headers=response_headers,
+ headers=response_headers,
+ status=status_code,
+ spec=["_headers", "content", "headers", "status"],
+ )
+
+ @pytest.mark.asyncio
+ async def test_consume_next_chunk_already_finished(self):
+ download = download_mod.RawChunkedDownload(sync_test.EXAMPLE_URL, 512, None)
+ download._finished = True
+ with pytest.raises(ValueError):
+ await download.consume_next_chunk(None)
+
+ def _mock_transport(self, start, chunk_size, total_bytes, content=b""):
+ transport = mock.AsyncMock(spec=["request"])
+ assert len(content) == chunk_size
+ mockResponse = self._mock_response(
+ start,
+ start + chunk_size - 1,
+ total_bytes,
+ content=content,
+ status_code=int(http.client.OK),
+ )
+ transport.request = mock.AsyncMock(spec=["__call__"], return_value=mockResponse)
+
+ return transport
+
+ @pytest.mark.asyncio
+ async def test_consume_next_chunk(self):
+ start = 1536
+ stream = io.BytesIO()
+ data = b"Just one chunk."
+ chunk_size = len(data)
+ download = download_mod.RawChunkedDownload(
+ sync_test.EXAMPLE_URL, chunk_size, stream, start=start
+ )
+ total_bytes = 16384
+ transport = self._mock_transport(start, chunk_size, total_bytes, content=data)
+
+ # Verify the internal state before consuming a chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually consume the chunk and check the output.
+ ret_val = await download.consume_next_chunk(transport)
+ assert ret_val is transport.request.return_value
+ range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1)
+ download_headers = {"range": range_bytes}
+ transport.request.assert_called_once_with(
+ "GET",
+ sync_test.EXAMPLE_URL,
+ data=None,
+ headers=download_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+ assert stream.getvalue() == data
+
+ # Go back and check the internal state after consuming the chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == chunk_size
+ assert download.total_bytes == total_bytes
+
+ @pytest.mark.asyncio
+ async def test_consume_next_chunk_with_custom_timeout(self):
+ start = 1536
+ stream = io.BytesIO()
+ data = b"Just one chunk."
+ chunk_size = len(data)
+ download = download_mod.RawChunkedDownload(
+ sync_test.EXAMPLE_URL, chunk_size, stream, start=start
+ )
+ total_bytes = 16384
+ transport = self._mock_transport(start, chunk_size, total_bytes, content=data)
+
+ # Actually consume the chunk and check the output.
+ await download.consume_next_chunk(transport, timeout=14.7)
+
+ range_bytes = "bytes={:d}-{:d}".format(start, start + chunk_size - 1)
+ download_headers = {"range": range_bytes}
+ transport.request.assert_called_once_with(
+ "GET",
+ sync_test.EXAMPLE_URL,
+ data=None,
+ headers=download_headers,
+ timeout=14.7,
+ )
+
+ assert stream.getvalue() == data
+
+ # Go back and check the internal state after consuming the chunk.
+ assert not download.finished
+ assert download.bytes_downloaded == chunk_size
+ assert download.total_bytes == total_bytes
+
+
+class Test__add_decoder(object):
+ def test_non_gzipped(self):
+ response_raw = mock.AsyncMock(headers={}, spec=["headers"])
+ md5_hash = download_mod._add_decoder(response_raw, mock.sentinel.md5_hash)
+
+ assert md5_hash is mock.sentinel.md5_hash
+
+ def test_gzipped(self):
+ headers = {"content-encoding": "gzip"}
+ response_raw = mock.AsyncMock(headers=headers, spec=["headers", "_decoder"])
+ md5_hash = download_mod._add_decoder(response_raw, mock.sentinel.md5_hash)
+
+ assert md5_hash is not mock.sentinel.md5_hash
+
+ assert isinstance(md5_hash, _helpers._DoNothingHash)
+ assert isinstance(response_raw._decoder, download_mod._GzipDecoder)
+ assert response_raw._decoder._checksum is mock.sentinel.md5_hash
+
+
+class Test_GzipDecoder(object):
+ def test_constructor(self):
+ decoder = download_mod._GzipDecoder(mock.sentinel.md5_hash)
+ assert decoder._checksum is mock.sentinel.md5_hash
+
+ def test_decompress(self):
+ md5_hash = mock.Mock(spec=["update"])
+ decoder = download_mod._GzipDecoder(md5_hash)
+
+ data = b"\x1f\x8b\x08\x08"
+ result = decoder.decompress(data)
+
+ assert result == b""
+ md5_hash.update.assert_called_once_with(data)
+
+ def test_decompress_with_max_length(self):
+ md5_hash = mock.Mock(spec=["update"])
+ decoder = download_mod._GzipDecoder(md5_hash)
+
+ with mock.patch.object(
+ type(decoder).__bases__[0], "decompress"
+ ) as mock_super_decompress:
+ mock_super_decompress.return_value = b"decompressed"
+ data = b"\x1f\x8b\x08\x08"
+ result = decoder.decompress(data, max_length=10)
+
+ assert result == b"decompressed"
+ md5_hash.update.assert_called_once_with(data)
+ mock_super_decompress.assert_called_once_with(data, max_length=10)
+
+
+class AsyncIter:
+ def __init__(self, items):
+ self.items = items
+
+ async def __aiter__(self):
+ for item in self.items:
+ yield item
+
+
+def _mock_response(status=http.client.OK, chunks=(), headers=None):
+ if headers is None:
+ headers = {}
+
+ if chunks:
+ chunklist = b"".join(chunks)
+ stream_content = mock.AsyncMock(spec=["__call__", "read", "iter_chunked"])
+ stream_content.read = mock.AsyncMock(spec=["__call__"], return_value=chunklist)
+ stream_content.iter_chunked.return_value = AsyncIter(chunks)
+ mock_raw = mock.AsyncMock(headers=headers, spec=["headers"])
+ response = mock.AsyncMock(
+ _headers=headers,
+ headers=headers,
+ status=int(status),
+ raw=mock_raw,
+ content=stream_content,
+ spec=[
+ "__aenter__",
+ "__aexit__",
+ "_headers",
+ "iter_chunked",
+ "status",
+ "headers",
+ "raw",
+ "content",
+ ],
+ )
+ # i.e. context manager returns ``self``.
+ response.__aenter__.return_value = response
+ response.__aexit__.return_value = None
+ return response
+ else:
+ return mock.AsyncMock(
+ _headers=headers,
+ headers=headers,
+ status=int(status),
+ spec=["_headers", "status", "headers"],
+ )
+
+
+def _mock_raw_response(status_code=http.client.OK, chunks=(), headers=None):
+ if headers is None:
+ headers = {}
+ chunklist = b"".join(chunks)
+ stream_content = mock.AsyncMock(spec=["__call__", "read", "iter_chunked"])
+ stream_content.read = mock.AsyncMock(spec=["__call__"], return_value=chunklist)
+ stream_content.iter_chunked.return_value = AsyncIter(chunks)
+ mock_raw = mock.AsyncMock(_headers=headers, headers=headers, spec=["__call__"])
+ response = mock.AsyncMock(
+ _headers=headers,
+ headers=headers,
+ status=int(status_code),
+ raw=mock_raw,
+ content=stream_content,
+ spec=[
+ "__aenter__",
+ "__aexit__",
+ "_headers",
+ "iter_chunked",
+ "status",
+ "headers",
+ "raw",
+ "content",
+ ],
+ )
+ # i.e. context manager returns ``self``.
+ response.__aenter__.return_value = response
+ response.__aexit__.return_value = None
+ return response
diff --git a/packages/google-resumable-media/tests_async/unit/requests/test_upload.py b/packages/google-resumable-media/tests_async/unit/requests/test_upload.py
new file mode 100644
index 000000000000..e9a0eb1a8cc6
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/unit/requests/test_upload.py
@@ -0,0 +1,396 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+import json
+
+import aiohttp # type: ignore
+import mock # type: ignore
+import pytest # type: ignore
+
+import google._async_resumable_media.requests.upload as upload_mod
+from tests.unit.requests import test_upload as sync_test
+
+SIMPLE_URL = sync_test.SIMPLE_URL
+MULTIPART_URL = sync_test.MULTIPART_URL
+RESUMABLE_URL = sync_test.RESUMABLE_URL
+ONE_MB = sync_test.ONE_MB
+BASIC_CONTENT = sync_test.BASIC_CONTENT
+JSON_TYPE = sync_test.JSON_TYPE
+JSON_TYPE_LINE = sync_test.JSON_TYPE_LINE
+EXPECTED_TIMEOUT = aiohttp.ClientTimeout(
+ total=None, connect=61, sock_read=60, sock_connect=None
+)
+
+
+class TestSimpleUpload(object):
+ @pytest.mark.asyncio
+ async def test_transmit(self):
+ data = b"I have got a lovely bunch of coconuts."
+ content_type = BASIC_CONTENT
+ upload = upload_mod.SimpleUpload(SIMPLE_URL)
+
+ transport = mock.AsyncMock(spec=["request"])
+ transport.request = mock.AsyncMock(
+ spec=["__call__"], return_value=_make_response()
+ )
+
+ assert not upload.finished
+
+ ret_val = await upload.transmit(transport, data, content_type)
+
+ assert ret_val is transport.request.return_value
+
+ upload_headers = {"content-type": content_type}
+ transport.request.assert_called_once_with(
+ "POST",
+ SIMPLE_URL,
+ data=data,
+ headers=upload_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ assert upload.finished
+
+ @pytest.mark.asyncio
+ async def test_transmit_w_custom_timeout(self):
+ data = b"I have got a lovely bunch of coconuts."
+ content_type = BASIC_CONTENT
+ upload = upload_mod.SimpleUpload(SIMPLE_URL)
+
+ transport = mock.AsyncMock(spec=["request"])
+ transport.request = mock.AsyncMock(
+ spec=["__call__"], return_value=_make_response()
+ )
+
+ assert not upload.finished
+
+ ret_val = await upload.transmit(transport, data, content_type, timeout=12.6)
+
+ assert ret_val is transport.request.return_value
+
+ upload_headers = {"content-type": content_type}
+ transport.request.assert_called_once_with(
+ "POST",
+ SIMPLE_URL,
+ data=data,
+ headers=upload_headers,
+ timeout=12.6,
+ )
+
+ assert upload.finished
+
+
+class TestMultipartUpload(object):
+ @mock.patch(
+ "google._async_resumable_media._upload.get_boundary", return_value=b"==4=="
+ )
+ @pytest.mark.asyncio
+ async def test_transmit(self, mock_get_boundary):
+ data = b"Mock data here and there."
+ metadata = {"Hey": "You", "Guys": "90909"}
+ content_type = BASIC_CONTENT
+ upload = upload_mod.MultipartUpload(MULTIPART_URL)
+
+ transport = mock.AsyncMock(spec=["request"])
+ transport.request = mock.AsyncMock(
+ spec=["__call__"], return_value=_make_response()
+ )
+
+ assert not upload.finished
+
+ ret_val = await upload.transmit(transport, data, metadata, content_type)
+
+ assert ret_val is transport.request.return_value
+
+ expected_payload = (
+ b"--==4==\r\n"
+ + JSON_TYPE_LINE
+ + b"\r\n"
+ + json.dumps(metadata).encode("utf-8")
+ + b"\r\n"
+ + b"--==4==\r\n"
+ b"content-type: text/plain\r\n"
+ b"\r\n"
+ b"Mock data here and there.\r\n"
+ b"--==4==--"
+ )
+ multipart_type = b'multipart/related; boundary="==4=="'
+ upload_headers = {"content-type": multipart_type}
+ transport.request.assert_called_once_with(
+ "POST",
+ MULTIPART_URL,
+ data=expected_payload,
+ headers=upload_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+ assert upload.finished
+ mock_get_boundary.assert_called_once_with()
+
+ @mock.patch(
+ "google._async_resumable_media._upload.get_boundary", return_value=b"==4=="
+ )
+ @pytest.mark.asyncio
+ async def test_transmit_w_custom_timeout(self, mock_get_boundary):
+ data = b"Mock data here and there."
+ metadata = {"Hey": "You", "Guys": "90909"}
+ content_type = BASIC_CONTENT
+ upload = upload_mod.MultipartUpload(MULTIPART_URL)
+
+ transport = mock.AsyncMock(spec=["request"])
+ transport.request = mock.AsyncMock(
+ spec=["__call__"], return_value=_make_response()
+ )
+
+ await upload.transmit(transport, data, metadata, content_type, timeout=12.6)
+
+ expected_payload = (
+ b"--==4==\r\n"
+ + JSON_TYPE_LINE
+ + b"\r\n"
+ + json.dumps(metadata).encode("utf-8")
+ + b"\r\n"
+ + b"--==4==\r\n"
+ b"content-type: text/plain\r\n"
+ b"\r\n"
+ b"Mock data here and there.\r\n"
+ b"--==4==--"
+ )
+ multipart_type = b'multipart/related; boundary="==4=="'
+ upload_headers = {"content-type": multipart_type}
+
+ transport.request.assert_called_once_with(
+ "POST",
+ MULTIPART_URL,
+ data=expected_payload,
+ headers=upload_headers,
+ timeout=12.6,
+ )
+ assert upload.finished
+ mock_get_boundary.assert_called_once_with()
+
+
+class TestResumableUpload(object):
+ @pytest.mark.asyncio
+ async def test_initiate(self):
+ upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ data = b"Knock knock who is there"
+ stream = io.BytesIO(data)
+ metadata = {"name": "got-jokes.txt"}
+
+ transport = mock.AsyncMock(spec=["request"])
+ location = ("http://test.invalid?upload_id=AACODBBBxuw9u3AA",)
+ response_headers = {"location": location}
+ transport.request = mock.AsyncMock(
+ spec=["__call__"], return_value=_make_response(headers=response_headers)
+ )
+
+ # Check resumable_url before.
+ assert upload._resumable_url is None
+ # Make request and check the return value (against the mock).
+ total_bytes = 100
+ assert total_bytes > len(data)
+ response = await upload.initiate(
+ transport,
+ stream,
+ metadata,
+ BASIC_CONTENT,
+ total_bytes=total_bytes,
+ stream_final=False,
+ )
+ assert response is transport.request.return_value
+ # Check resumable_url after.
+ assert upload._resumable_url == location
+ # Make sure the mock was called as expected.
+ json_bytes = b'{"name": "got-jokes.txt"}'
+ expected_headers = {
+ "content-type": JSON_TYPE,
+ "x-upload-content-type": BASIC_CONTENT,
+ "x-upload-content-length": "{:d}".format(total_bytes),
+ }
+ transport.request.assert_called_once_with(
+ "POST",
+ RESUMABLE_URL,
+ data=json_bytes,
+ headers=expected_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ @pytest.mark.asyncio
+ async def test_initiate_w_custom_timeout(self):
+ upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ data = b"Knock knock who is there"
+ stream = io.BytesIO(data)
+ metadata = {"name": "got-jokes.txt"}
+
+ transport = mock.AsyncMock(spec=["request"])
+ location = ("http://test.invalid?upload_id=AACODBBBxuw9u3AA",)
+ response_headers = {"location": location}
+ transport.request = mock.AsyncMock(
+ spec=["__call__"], return_value=_make_response(headers=response_headers)
+ )
+
+ # Check resumable_url before.
+ assert upload._resumable_url is None
+ # Make request and check the return value (against the mock).
+ total_bytes = 100
+ assert total_bytes > len(data)
+ response = await upload.initiate(
+ transport,
+ stream,
+ metadata,
+ BASIC_CONTENT,
+ total_bytes=total_bytes,
+ stream_final=False,
+ timeout=12.6,
+ )
+ assert response is transport.request.return_value
+ # Check resumable_url after.
+ assert upload._resumable_url == location
+ # Make sure the mock was called as expected.
+ json_bytes = b'{"name": "got-jokes.txt"}'
+ expected_headers = {
+ "content-type": JSON_TYPE,
+ "x-upload-content-type": BASIC_CONTENT,
+ "x-upload-content-length": "{:d}".format(total_bytes),
+ }
+ transport.request.assert_called_once_with(
+ "POST",
+ RESUMABLE_URL,
+ data=json_bytes,
+ headers=expected_headers,
+ timeout=12.6,
+ )
+
+ @staticmethod
+ def _upload_in_flight(data, headers=None):
+ upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB, headers=headers)
+ upload._stream = io.BytesIO(data)
+ upload._content_type = BASIC_CONTENT
+ upload._total_bytes = len(data)
+ upload._resumable_url = "http://test.invalid?upload_id=not-none"
+ return upload
+
+ @staticmethod
+ def _chunk_mock(status_code, response_headers):
+ transport = mock.AsyncMock(spec=["request"])
+ put_response = _make_response(status_code=status_code, headers=response_headers)
+ transport.request = mock.AsyncMock(spec=["__call__"], return_value=put_response)
+
+ return transport
+
+ @pytest.mark.asyncio
+ async def test_transmit_next_chunk(self):
+ data = b"This time the data is official."
+ upload = self._upload_in_flight(data)
+ # Make a fake chunk size smaller than 256 KB.
+ chunk_size = 10
+ assert chunk_size < len(data)
+ upload._chunk_size = chunk_size
+ # Make a fake 308 response.
+ response_headers = {"range": "bytes=0-{:d}".format(chunk_size - 1)}
+ transport = self._chunk_mock(http.client.PERMANENT_REDIRECT, response_headers)
+ # Check the state before the request.
+ assert upload._bytes_uploaded == 0
+
+ # Make request and check the return value (against the mock).
+ response = await upload.transmit_next_chunk(transport)
+ assert response is transport.request.return_value
+ # Check that the state has been updated.
+ assert upload._bytes_uploaded == chunk_size
+ # Make sure the mock was called as expected.
+ payload = data[:chunk_size]
+ content_range = "bytes 0-{:d}/{:d}".format(chunk_size - 1, len(data))
+ expected_headers = {
+ "content-range": content_range,
+ "content-type": BASIC_CONTENT,
+ }
+ transport.request.assert_called_once_with(
+ "PUT",
+ upload.resumable_url,
+ data=payload,
+ headers=expected_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+ @pytest.mark.asyncio
+ async def test_transmit_next_chunk_w_custom_timeout(self):
+ data = b"This time the data is official."
+ upload = self._upload_in_flight(data)
+ # Make a fake chunk size smaller than 256 KB.
+ chunk_size = 10
+ assert chunk_size < len(data)
+ upload._chunk_size = chunk_size
+ # Make a fake 308 response.
+ response_headers = {"range": "bytes=0-{:d}".format(chunk_size - 1)}
+ transport = self._chunk_mock(http.client.PERMANENT_REDIRECT, response_headers)
+ # Check the state before the request.
+ assert upload._bytes_uploaded == 0
+
+ # Make request and check the return value (against the mock).
+ response = await upload.transmit_next_chunk(transport, timeout=12.6)
+ assert response is transport.request.return_value
+ # Check that the state has been updated.
+ assert upload._bytes_uploaded == chunk_size
+ # Make sure the mock was called as expected.
+ payload = data[:chunk_size]
+ content_range = "bytes 0-{:d}/{:d}".format(chunk_size - 1, len(data))
+ expected_headers = {
+ "content-range": content_range,
+ "content-type": BASIC_CONTENT,
+ }
+ transport.request.assert_called_once_with(
+ "PUT",
+ upload.resumable_url,
+ data=payload,
+ headers=expected_headers,
+ timeout=12.6,
+ )
+
+ @pytest.mark.asyncio
+ async def test_recover(self):
+ upload = upload_mod.ResumableUpload(RESUMABLE_URL, ONE_MB)
+ upload._invalid = True # Make sure invalid.
+ upload._stream = mock.Mock(spec=["seek"])
+ upload._resumable_url = "http://test.invalid?upload_id=big-deal"
+
+ end = 55555
+ headers = {"range": "bytes=0-{:d}".format(end)}
+ transport = self._chunk_mock(http.client.PERMANENT_REDIRECT, headers)
+
+ ret_val = await upload.recover(transport)
+ assert ret_val is transport.request.return_value
+ # Check the state of ``upload`` after.
+ assert upload.bytes_uploaded == end + 1
+ assert not upload.invalid
+ upload._stream.seek.assert_called_once_with(end + 1)
+ expected_headers = {"content-range": "bytes */*"}
+ transport.request.assert_called_once_with(
+ "PUT",
+ upload.resumable_url,
+ data=None,
+ headers=expected_headers,
+ timeout=EXPECTED_TIMEOUT,
+ )
+
+
+def _make_response(status_code=200, headers=None):
+ headers = headers or {}
+ return mock.Mock(
+ _headers=headers,
+ headers=headers,
+ status=status_code,
+ spec=["_headers", "headers", "status_code"],
+ )
diff --git a/packages/google-resumable-media/tests_async/unit/test__download.py b/packages/google-resumable-media/tests_async/unit/test__download.py
new file mode 100644
index 000000000000..8dfd13040a8e
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/unit/test__download.py
@@ -0,0 +1,776 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+
+import google.auth.transport._aiohttp_requests as aiohttp_requests # type: ignore
+import mock
+import pytest # type: ignore
+
+from google._async_resumable_media import _download
+from google.resumable_media import common
+from tests.unit import test__download as sync_test
+
+
+EXAMPLE_URL = sync_test.EXAMPLE_URL
+
+
+class TestDownloadBase(object):
+ def test_constructor_defaults(self):
+ download = _download.DownloadBase(EXAMPLE_URL)
+ assert download.media_url == EXAMPLE_URL
+ assert download._stream is None
+ assert download.start is None
+ assert download.end is None
+ assert download._headers == {}
+ assert not download._finished
+ _check_retry_strategy(download)
+
+ def test_constructor_explicit(self):
+ start = 11
+ end = 10001
+ headers = {"foof": "barf"}
+ download = _download.DownloadBase(
+ EXAMPLE_URL,
+ stream=mock.sentinel.stream,
+ start=start,
+ end=end,
+ headers=headers,
+ )
+ assert download.media_url == EXAMPLE_URL
+ assert download._stream is mock.sentinel.stream
+ assert download.start == start
+ assert download.end == end
+ assert download._headers is headers
+ assert not download._finished
+ _check_retry_strategy(download)
+
+ def test_finished_property(self):
+ download = _download.DownloadBase(EXAMPLE_URL)
+ # Default value of @property.
+ assert not download.finished
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ download.finished = False
+
+ # Set it privately and then check the @property.
+ download._finished = True
+ assert download.finished
+
+ def test__get_status_code(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _download.DownloadBase._get_status_code(None)
+
+ exc_info.match("virtual")
+
+ def test__get_headers(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _download.DownloadBase._get_headers(None)
+
+ exc_info.match("virtual")
+
+ def test__get_body(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _download.DownloadBase._get_body(None)
+
+ exc_info.match("virtual")
+
+
+class TestDownload(object):
+ def test__prepare_request_already_finished(self):
+ download = _download.Download(EXAMPLE_URL)
+ download._finished = True
+ with pytest.raises(ValueError):
+ download._prepare_request()
+
+ def test__prepare_request(self):
+ download1 = _download.Download(EXAMPLE_URL)
+ method1, url1, payload1, headers1 = download1._prepare_request()
+ assert method1 == "GET"
+ assert url1 == EXAMPLE_URL
+ assert payload1 is None
+ assert headers1 == {}
+
+ download2 = _download.Download(EXAMPLE_URL, start=53)
+ method2, url2, payload2, headers2 = download2._prepare_request()
+ assert method2 == "GET"
+ assert url2 == EXAMPLE_URL
+ assert payload2 is None
+ assert headers2 == {"range": "bytes=53-"}
+
+ def test__prepare_request_with_headers(self):
+ headers = {"spoonge": "borb"}
+ download = _download.Download(EXAMPLE_URL, start=11, end=111, headers=headers)
+ method, url, payload, new_headers = download._prepare_request()
+ assert method == "GET"
+ assert url == EXAMPLE_URL
+ assert payload is None
+ assert new_headers is headers
+ assert headers == {"range": "bytes=11-111", "spoonge": "borb"}
+
+ @pytest.mark.asyncio
+ async def test__process_response(self):
+ download = _download.Download(EXAMPLE_URL)
+ _fix_up_virtual(download)
+
+ # Make sure **not finished** before.
+ assert not download.finished
+ response = mock.AsyncMock(status=int(http.client.OK), spec=["status"])
+ ret_val = download._process_response(response)
+ assert ret_val is None
+ # Make sure **finished** after.
+ assert download.finished
+
+ @pytest.mark.asyncio
+ async def test__process_response_bad_status(self):
+ download = _download.Download(EXAMPLE_URL)
+ _fix_up_virtual(download)
+
+ # Make sure **not finished** before.
+ assert not download.finished
+ response = mock.AsyncMock(status=int(http.client.NOT_FOUND), spec=["status"])
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == response.status
+ assert error.args[3] == http.client.OK
+ assert error.args[4] == http.client.PARTIAL_CONTENT
+ # Make sure **finished** even after a failure.
+ assert download.finished
+
+ def test_consume(self):
+ download = _download.Download(EXAMPLE_URL)
+ with pytest.raises(NotImplementedError) as exc_info:
+ download.consume(None)
+
+ exc_info.match("virtual")
+
+
+class TestChunkedDownload(object):
+ def test_constructor_defaults(self):
+ chunk_size = 256
+ stream = mock.sentinel.stream
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ assert download.media_url == EXAMPLE_URL
+ assert download.chunk_size == chunk_size
+ assert download.start == 0
+ assert download.end is None
+ assert download._headers == {}
+ assert not download._finished
+ _check_retry_strategy(download)
+ assert download._stream is stream
+ assert download._bytes_downloaded == 0
+ assert download._total_bytes is None
+ assert not download._invalid
+
+ def test_constructor_bad_start(self):
+ with pytest.raises(ValueError):
+ _download.ChunkedDownload(EXAMPLE_URL, 256, None, start=-11)
+
+ def test_bytes_downloaded_property(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ # Default value of @property.
+ assert download.bytes_downloaded == 0
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ download.bytes_downloaded = 1024
+
+ # Set it privately and then check the @property.
+ download._bytes_downloaded = 128
+ assert download.bytes_downloaded == 128
+
+ def test_total_bytes_property(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ # Default value of @property.
+ assert download.total_bytes is None
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ download.total_bytes = 65536
+
+ # Set it privately and then check the @property.
+ download._total_bytes = 8192
+ assert download.total_bytes == 8192
+
+ def test__get_byte_range(self):
+ chunk_size = 512
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
+ curr_start, curr_end = download._get_byte_range()
+ assert curr_start == 0
+ assert curr_end == chunk_size - 1
+
+ def test__get_byte_range_with_end(self):
+ chunk_size = 512
+ start = 1024
+ end = 1151
+ download = _download.ChunkedDownload(
+ EXAMPLE_URL, chunk_size, None, start=start, end=end
+ )
+ curr_start, curr_end = download._get_byte_range()
+ assert curr_start == start
+ assert curr_end == end
+ # Make sure this is less than the chunk size.
+ actual_size = curr_end - curr_start + 1
+ assert actual_size < chunk_size
+
+ def test__get_byte_range_with_total_bytes(self):
+ chunk_size = 512
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
+ total_bytes = 207
+ download._total_bytes = total_bytes
+ curr_start, curr_end = download._get_byte_range()
+ assert curr_start == 0
+ assert curr_end == total_bytes - 1
+ # Make sure this is less than the chunk size.
+ actual_size = curr_end - curr_start + 1
+ assert actual_size < chunk_size
+
+ @staticmethod
+ def _response_content_range(start_byte, end_byte, total_bytes):
+ return "bytes {:d}-{:d}/{:d}".format(start_byte, end_byte, total_bytes)
+
+ def _response_headers(self, start_byte, end_byte, total_bytes):
+ content_length = end_byte - start_byte + 1
+ resp_range = self._response_content_range(start_byte, end_byte, total_bytes)
+ return {
+ "content-length": "{:d}".format(content_length),
+ "content-range": resp_range,
+ }
+
+ def _mock_response(
+ self, start_byte, end_byte, total_bytes, content=None, status_code=None
+ ):
+ response_headers = self._response_headers(start_byte, end_byte, total_bytes)
+ content_stream = mock.AsyncMock(spec=["__call__", "read"])
+ content_stream.read = mock.AsyncMock(spec=["__call__"], return_value=content)
+ return mock.AsyncMock(
+ content=content_stream,
+ headers=response_headers,
+ status=status_code,
+ spec=["__call__", "_content", "headers", "status"],
+ )
+
+ def test__prepare_request_already_finished(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 64, None)
+ download._finished = True
+ with pytest.raises(ValueError) as exc_info:
+ download._prepare_request()
+
+ assert exc_info.match("Download has finished.")
+
+ def test__prepare_request_invalid(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 64, None)
+ download._invalid = True
+ with pytest.raises(ValueError) as exc_info:
+ download._prepare_request()
+
+ assert exc_info.match("Download is invalid and cannot be re-used.")
+
+ def test__prepare_request(self):
+ chunk_size = 2048
+ download1 = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, None)
+ method1, url1, payload1, headers1 = download1._prepare_request()
+ assert method1 == "GET"
+ assert url1 == EXAMPLE_URL
+ assert payload1 is None
+ assert headers1 == {"range": "bytes=0-2047"}
+
+ download2 = _download.ChunkedDownload(
+ EXAMPLE_URL, chunk_size, None, start=19991
+ )
+ download2._total_bytes = 20101
+ method2, url2, payload2, headers2 = download2._prepare_request()
+ assert method2 == "GET"
+ assert url2 == EXAMPLE_URL
+ assert payload2 is None
+ assert headers2 == {"range": "bytes=19991-20100"}
+
+ def test__prepare_request_with_headers(self):
+ chunk_size = 2048
+ headers = {"patrizio": "Starf-ish"}
+ download = _download.ChunkedDownload(
+ EXAMPLE_URL, chunk_size, None, headers=headers
+ )
+ method, url, payload, new_headers = download._prepare_request()
+ assert method == "GET"
+ assert url == EXAMPLE_URL
+ assert payload is None
+ assert new_headers is headers
+ expected = {"patrizio": "Starf-ish", "range": "bytes=0-2047"}
+ assert headers == expected
+
+ def test__make_invalid(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 512, None)
+ assert not download.invalid
+ download._make_invalid()
+ assert download.invalid
+
+ @pytest.mark.asyncio
+ async def test__process_response(self):
+ data = b"1234xyztL" * 37 # 9 * 37 == 33
+ chunk_size = len(data)
+ stream = io.BytesIO()
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ already = 22
+ download._bytes_downloaded = already
+ total_bytes = 4444
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == already
+ assert download.total_bytes is None
+ # Actually call the method to update.
+ response = self._mock_response(
+ already,
+ already + chunk_size - 1,
+ total_bytes,
+ content=data,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ )
+
+ await download._process_response(response)
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == already + chunk_size
+ assert download.total_bytes == total_bytes
+ assert stream.getvalue() == data
+
+ @pytest.mark.asyncio
+ async def test__process_response_transfer_encoding(self):
+ data = b"1234xyztL" * 37
+ chunk_size = len(data)
+ stream = io.BytesIO()
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ already = 22
+ download._bytes_downloaded = already
+ total_bytes = 4444
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == already
+ assert download.total_bytes is None
+ assert not download.invalid
+ # Actually call the method to update.
+ response = self._mock_response(
+ already,
+ already + chunk_size - 1,
+ total_bytes,
+ content=data,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ )
+ response.headers["transfer-encoding"] = "chunked"
+ del response.headers["content-length"]
+ await download._process_response(response)
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == already + chunk_size
+ assert download.total_bytes == total_bytes
+ assert stream.getvalue() == data
+
+ @pytest.mark.asyncio
+ async def test__process_response_bad_status(self):
+ chunk_size = 384
+ stream = mock.Mock(spec=["write"])
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ total_bytes = 300
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually call the method to update.
+ response = self._mock_response(
+ 0, total_bytes - 1, total_bytes, status_code=int(http.client.NOT_FOUND)
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == response.status
+ assert error.args[3] == http.client.OK
+ assert error.args[4] == http.client.PARTIAL_CONTENT
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert download.invalid
+ stream.write.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test__process_response_missing_content_length(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ _fix_up_virtual(download)
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert not download.invalid
+ # Actually call the method to update.
+ content_stream = mock.AsyncMock(spec=["__call", "read"])
+ content_stream.read = mock.AsyncMock(
+ spec=["__call__"], return_value=b"DEADBEEF"
+ )
+ response = mock.AsyncMock(
+ headers={"content-range": "bytes 0-99/99"},
+ status=int(http.client.PARTIAL_CONTENT),
+ content=content_stream,
+ spec=["headers", "status", "content"],
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 2
+ assert error.args[1] == "content-length"
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert download.invalid
+
+ @pytest.mark.asyncio
+ async def test__process_response_bad_content_range(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ _fix_up_virtual(download)
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert not download.invalid
+ # Actually call the method to update.
+ data = b"stuff"
+ headers = {
+ "content-length": "{:d}".format(len(data)),
+ "content-range": "kites x-y/58",
+ }
+ content_stream = mock.AsyncMock(spec=["__call", "read"])
+ content_stream.read = mock.AsyncMock(spec=["__call__"], return_value=data)
+ response = mock.AsyncMock(
+ content=content_stream,
+ headers=headers,
+ status=int(http.client.PARTIAL_CONTENT),
+ spec=["content", "headers", "status"],
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == headers["content-range"]
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert download.invalid
+
+ @pytest.mark.asyncio
+ async def test__process_response_body_wrong_length(self):
+ chunk_size = 10
+ stream = mock.Mock(spec=["write"])
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ total_bytes = 100
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually call the method to update.
+ data = b"not 10"
+ response = self._mock_response(
+ 0,
+ chunk_size - 1,
+ total_bytes,
+ content=data,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await download._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[2] == chunk_size
+ assert error.args[4] == len(data)
+ # Check internal state after.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ assert download.invalid
+ stream.write.assert_not_called()
+
+ @pytest.mark.asyncio
+ async def test__process_response_when_finished(self):
+ chunk_size = 256
+ stream = io.BytesIO()
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ total_bytes = 200
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+ # Actually call the method to update.
+ data = b"abcd" * 50 # 4 * 50 == 200
+ response = self._mock_response(
+ 0,
+ total_bytes - 1,
+ total_bytes,
+ content=data,
+ status_code=int(http.client.OK),
+ )
+ await download._process_response(response)
+ # Check internal state after.
+ assert download.finished
+ assert download.bytes_downloaded == total_bytes
+ assert total_bytes < chunk_size
+ assert download.total_bytes == total_bytes
+ assert stream.getvalue() == data
+
+ @pytest.mark.asyncio
+ async def test__process_response_when_reaching_end(self):
+ chunk_size = 8192
+ end = 65000
+ stream = io.BytesIO()
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream, end=end)
+ _fix_up_virtual(download)
+
+ download._bytes_downloaded = 7 * chunk_size
+ download._total_bytes = 8 * chunk_size
+
+ # Check internal state before.
+ assert not download.finished
+ assert download.bytes_downloaded == 7 * chunk_size
+ assert download.total_bytes == 8 * chunk_size
+ # Actually call the method to update.
+ expected_size = end - 7 * chunk_size + 1
+ data = b"B" * expected_size
+ response = self._mock_response(
+ 7 * chunk_size,
+ end,
+ 8 * chunk_size,
+ content=data,
+ status_code=int(http.client.PARTIAL_CONTENT),
+ )
+ await download._process_response(response)
+ # Check internal state after.
+ assert download.finished
+ assert download.bytes_downloaded == end + 1
+ assert download.bytes_downloaded < download.total_bytes
+ assert download.total_bytes == 8 * chunk_size
+ assert stream.getvalue() == data
+
+ @pytest.mark.asyncio
+ async def test__process_response_when_content_range_is_zero(self):
+ chunk_size = 10
+ stream = mock.AsyncMock(spec=["write"])
+ download = _download.ChunkedDownload(EXAMPLE_URL, chunk_size, stream)
+ _fix_up_virtual(download)
+
+ content_range = _download._ZERO_CONTENT_RANGE_HEADER
+ headers = {"content-range": content_range}
+ status_code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE
+ response = mock.AsyncMock(
+ headers=headers, status=status_code, spec=["headers", "status"]
+ )
+ await download._process_response(response)
+ stream.write.assert_not_called()
+ assert download.finished
+ assert download.bytes_downloaded == 0
+ assert download.total_bytes is None
+
+ def test_consume_next_chunk(self):
+ download = _download.ChunkedDownload(EXAMPLE_URL, 256, None)
+ with pytest.raises(NotImplementedError) as exc_info:
+ download.consume_next_chunk(None)
+
+ exc_info.match("virtual")
+
+
+class Test__add_bytes_range(object):
+ def test_do_nothing(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(None, None, headers)
+ assert ret_val is None
+ assert headers == {}
+
+ def test_both_vals(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(17, 1997, headers)
+ assert ret_val is None
+ assert headers == {"range": "bytes=17-1997"}
+
+ def test_end_only(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(None, 909, headers)
+ assert ret_val is None
+ assert headers == {"range": "bytes=0-909"}
+
+ def test_start_only(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(3735928559, None, headers)
+ assert ret_val is None
+ assert headers == {"range": "bytes=3735928559-"}
+
+ def test_start_as_offset(self):
+ headers = {}
+ ret_val = _download.add_bytes_range(-123454321, None, headers)
+ assert ret_val is None
+ assert headers == {"range": "bytes=-123454321"}
+
+
+class Test_get_range_info(object):
+ @staticmethod
+ def _make_response(content_range):
+ headers = {"content-range": content_range}
+ return mock.Mock(headers=headers, spec=["headers"])
+
+ def _success_helper(self, **kwargs):
+ content_range = "Bytes 7-11/42"
+ response = self._make_response(content_range)
+ start_byte, end_byte, total_bytes = _download.get_range_info(
+ response, _get_headers, **kwargs
+ )
+ assert start_byte == 7
+ assert end_byte == 11
+ assert total_bytes == 42
+
+ def test_success(self):
+ self._success_helper()
+
+ def test_success_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._success_helper(callback=callback)
+ callback.assert_not_called()
+
+ def _failure_helper(self, **kwargs):
+ content_range = "nope x-6/y"
+ response = self._make_response(content_range)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _download.get_range_info(response, _get_headers, **kwargs)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == content_range
+
+ def test_failure(self):
+ self._failure_helper()
+
+ def test_failure_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._failure_helper(callback=callback)
+ callback.assert_called_once_with()
+
+ def _missing_header_helper(self, **kwargs):
+ response = mock.Mock(headers={}, spec=["headers"])
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _download.get_range_info(response, _get_headers, **kwargs)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 2
+ assert error.args[1] == "content-range"
+
+ def test_missing_header(self):
+ self._missing_header_helper()
+
+ def test_missing_header_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._missing_header_helper(callback=callback)
+ callback.assert_called_once_with()
+
+
+class Test__check_for_zero_content_range(object):
+ @staticmethod
+ def _make_response(content_range, status_code):
+ headers = {"content-range": content_range}
+ return mock.AsyncMock(
+ headers=headers, status=status_code, spec=["headers", "status_code"]
+ )
+
+ def test_status_code_416_and_test_content_range_zero_both(self):
+ content_range = _download._ZERO_CONTENT_RANGE_HEADER
+ status_code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE
+ response = self._make_response(content_range, status_code)
+ assert _download._check_for_zero_content_range(
+ response, _get_status_code, _get_headers
+ )
+
+ def test_status_code_416_only(self):
+ content_range = "bytes 2-5/3"
+ status_code = http.client.REQUESTED_RANGE_NOT_SATISFIABLE
+ response = self._make_response(content_range, status_code)
+ assert not _download._check_for_zero_content_range(
+ response, _get_status_code, _get_headers
+ )
+
+ def test_content_range_zero_only(self):
+ content_range = _download._ZERO_CONTENT_RANGE_HEADER
+ status_code = http.client.OK
+ response = self._make_response(content_range, status_code)
+ assert not _download._check_for_zero_content_range(
+ response, _get_status_code, _get_headers
+ )
+
+
+def _get_status_code(response):
+ return response.status
+
+
+def _get_headers(response):
+ return response.headers
+
+
+async def _get_body(response):
+ # TODO(asyncio): This code differs from sync. Leaving this comment in case
+ # this difference causes downstream issues.
+ wrapped_response = aiohttp_requests._CombinedResponse(response)
+ content = await wrapped_response.raw_content()
+ return content
+
+
+def _fix_up_virtual(download):
+ download._get_status_code = _get_status_code
+ download._get_headers = _get_headers
+ download._get_body = _get_body
+
+
+def _check_retry_strategy(download):
+ retry_strategy = download._retry_strategy
+ assert isinstance(retry_strategy, common.RetryStrategy)
+ assert retry_strategy.max_sleep == common.MAX_SLEEP
+ assert retry_strategy.max_cumulative_retry == common.MAX_CUMULATIVE_RETRY
+ assert retry_strategy.max_retries is None
diff --git a/packages/google-resumable-media/tests_async/unit/test__helpers.py b/packages/google-resumable-media/tests_async/unit/test__helpers.py
new file mode 100644
index 000000000000..b95bb5a0ab18
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/unit/test__helpers.py
@@ -0,0 +1,305 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+
+import mock
+import pytest # type: ignore
+
+from google._async_resumable_media import _helpers
+from google.resumable_media import common
+
+
+def test_do_nothing():
+ ret_val = _helpers.do_nothing()
+ assert ret_val is None
+
+
+class Test_header_required(object):
+ def _success_helper(self, **kwargs):
+ name = "some-header"
+ value = "The Right Hand Side"
+ headers = {name: value, "other-name": "other-value"}
+ response = mock.Mock(
+ _headers=headers, headers=headers, spec=["_headers", "headers"]
+ )
+ result = _helpers.header_required(response, name, _get_headers, **kwargs)
+ assert result == value
+
+ def test_success(self):
+ self._success_helper()
+
+ def test_success_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._success_helper(callback=callback)
+ callback.assert_not_called()
+
+ def _failure_helper(self, **kwargs):
+ response = mock.Mock(_headers={}, headers={}, spec=["_headers", "headers"])
+ name = "any-name"
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _helpers.header_required(response, name, _get_headers, **kwargs)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 2
+ assert error.args[1] == name
+
+ def test_failure(self):
+ self._failure_helper()
+
+ def test_failure_with_callback(self):
+ callback = mock.Mock(spec=[])
+ self._failure_helper(callback=callback)
+ callback.assert_called_once_with()
+
+
+class Test_require_status_code(object):
+ @staticmethod
+ def _get_status_code(response):
+ return response.status_code
+
+ def test_success(self):
+ status_codes = (http.client.OK, http.client.CREATED)
+ acceptable = (
+ http.client.OK,
+ int(http.client.OK),
+ http.client.CREATED,
+ int(http.client.CREATED),
+ )
+ for value in acceptable:
+ response = _make_response(value)
+ status_code = _helpers.require_status_code(
+ response, status_codes, self._get_status_code
+ )
+ assert value == status_code
+
+ def test_success_with_callback(self):
+ status_codes = (http.client.OK,)
+ response = _make_response(http.client.OK)
+ callback = mock.Mock(spec=[])
+ status_code = _helpers.require_status_code(
+ response, status_codes, self._get_status_code, callback=callback
+ )
+ assert status_code == http.client.OK
+ callback.assert_not_called()
+
+ def test_failure(self):
+ status_codes = (http.client.CREATED, http.client.NO_CONTENT)
+ response = _make_response(http.client.OK)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _helpers.require_status_code(response, status_codes, self._get_status_code)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == response.status_code
+ assert error.args[3:] == status_codes
+
+ def test_failure_with_callback(self):
+ status_codes = (http.client.OK,)
+ response = _make_response(http.client.NOT_FOUND)
+ callback = mock.Mock(spec=[])
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ _helpers.require_status_code(
+ response, status_codes, self._get_status_code, callback=callback
+ )
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 4
+ assert error.args[1] == response.status_code
+ assert error.args[3:] == status_codes
+ callback.assert_called_once_with()
+
+
+class Test_calculate_retry_wait(object):
+ @mock.patch("random.randint", return_value=125)
+ def test_past_limit(self, randint_mock):
+ base_wait, wait_time = _helpers.calculate_retry_wait(70.0, 64.0)
+
+ assert base_wait == 64.0
+ assert wait_time == 64.125
+ randint_mock.assert_called_once_with(0, 1000)
+
+ @mock.patch("random.randint", return_value=250)
+ def test_at_limit(self, randint_mock):
+ base_wait, wait_time = _helpers.calculate_retry_wait(50.0, 50.0)
+
+ assert base_wait == 50.0
+ assert wait_time == 50.25
+ randint_mock.assert_called_once_with(0, 1000)
+
+ @mock.patch("random.randint", return_value=875)
+ def test_under_limit(self, randint_mock):
+ base_wait, wait_time = _helpers.calculate_retry_wait(16.0, 33.0)
+
+ assert base_wait == 32.0
+ assert wait_time == 32.875
+ randint_mock.assert_called_once_with(0, 1000)
+
+
+class Test_wait_and_retry(object):
+ @pytest.mark.asyncio
+ async def test_success_no_retry(self):
+ truthy = http.client.OK
+ assert truthy not in common.RETRYABLE
+ response = _make_response(truthy)
+
+ func = mock.AsyncMock(return_value=response, spec=[])
+ retry_strategy = common.RetryStrategy()
+ ret_val = await _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+
+ assert ret_val is response
+ func.assert_called_once_with()
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ @pytest.mark.asyncio
+ async def test_success_with_retry(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [125, 625, 375]
+
+ status_codes = (
+ http.client.INTERNAL_SERVER_ERROR,
+ http.client.BAD_GATEWAY,
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.NOT_FOUND,
+ )
+ responses = [_make_response(status_code) for status_code in status_codes]
+ func = mock.AsyncMock(side_effect=responses, spec=[])
+
+ retry_strategy = common.RetryStrategy()
+ ret_val = await _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+
+ assert ret_val == responses[-1]
+ assert status_codes[-1] not in common.RETRYABLE
+
+ assert func.call_count == 4
+ assert func.mock_calls == [mock.call()] * 4
+
+ assert randint_mock.call_count == 3
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3
+
+ assert sleep_mock.call_count == 3
+ sleep_mock.assert_any_call(1.125)
+ sleep_mock.assert_any_call(2.625)
+ sleep_mock.assert_any_call(4.375)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ @pytest.mark.asyncio
+ async def test_success_with_retry_connection_error(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [125, 625, 375]
+
+ response = _make_response(http.client.NOT_FOUND)
+ responses = [ConnectionError, ConnectionError, ConnectionError, response]
+ func = mock.AsyncMock(side_effect=responses, spec=[])
+
+ retry_strategy = common.RetryStrategy()
+ ret_val = await _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+
+ assert ret_val == responses[-1]
+
+ assert func.call_count == 4
+ assert func.mock_calls == [mock.call()] * 4
+
+ assert randint_mock.call_count == 3
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 3
+
+ assert sleep_mock.call_count == 3
+ sleep_mock.assert_any_call(1.125)
+ sleep_mock.assert_any_call(2.625)
+ sleep_mock.assert_any_call(4.375)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ @pytest.mark.asyncio
+ async def test_retry_exceeded_reraises_connection_error(
+ self, randint_mock, sleep_mock
+ ):
+ randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125]
+
+ responses = [ConnectionError] * 8
+ func = mock.AsyncMock(side_effect=responses, spec=[])
+
+ retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0)
+ with pytest.raises(ConnectionError):
+ await _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+
+ assert func.call_count == 8
+ assert func.mock_calls == [mock.call()] * 8
+
+ assert randint_mock.call_count == 7
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7
+
+ assert sleep_mock.call_count == 7
+ sleep_mock.assert_any_call(1.875)
+ sleep_mock.assert_any_call(2.0)
+ sleep_mock.assert_any_call(4.375)
+ sleep_mock.assert_any_call(8.5)
+ sleep_mock.assert_any_call(16.5)
+ sleep_mock.assert_any_call(32.25)
+ sleep_mock.assert_any_call(64.125)
+
+ @mock.patch("time.sleep")
+ @mock.patch("random.randint")
+ @pytest.mark.asyncio
+ async def test_retry_exceeds_max_cumulative(self, randint_mock, sleep_mock):
+ randint_mock.side_effect = [875, 0, 375, 500, 500, 250, 125]
+
+ status_codes = (
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.GATEWAY_TIMEOUT,
+ http.client.TOO_MANY_REQUESTS,
+ http.client.INTERNAL_SERVER_ERROR,
+ http.client.SERVICE_UNAVAILABLE,
+ http.client.BAD_GATEWAY,
+ http.client.GATEWAY_TIMEOUT,
+ http.client.TOO_MANY_REQUESTS,
+ )
+ responses = [_make_response(status_code) for status_code in status_codes]
+ func = mock.AsyncMock(side_effect=responses, spec=[])
+
+ retry_strategy = common.RetryStrategy(max_cumulative_retry=100.0)
+ ret_val = await _helpers.wait_and_retry(func, _get_status_code, retry_strategy)
+
+ assert ret_val == responses[-1]
+ assert status_codes[-1] in common.RETRYABLE
+
+ assert func.call_count == 8
+ assert func.mock_calls == [mock.call()] * 8
+
+ assert randint_mock.call_count == 7
+ assert randint_mock.mock_calls == [mock.call(0, 1000)] * 7
+
+ assert sleep_mock.call_count == 7
+ sleep_mock.assert_any_call(1.875)
+ sleep_mock.assert_any_call(2.0)
+ sleep_mock.assert_any_call(4.375)
+ sleep_mock.assert_any_call(8.5)
+ sleep_mock.assert_any_call(16.5)
+ sleep_mock.assert_any_call(32.25)
+ sleep_mock.assert_any_call(64.125)
+
+
+def _make_response(status_code):
+ return mock.AsyncMock(status_code=status_code, spec=["status_code"])
+
+
+def _get_status_code(response):
+ return response.status_code
+
+
+def _get_headers(response):
+ return response._headers
diff --git a/packages/google-resumable-media/tests_async/unit/test__upload.py b/packages/google-resumable-media/tests_async/unit/test__upload.py
new file mode 100644
index 000000000000..2f7d0e987c6e
--- /dev/null
+++ b/packages/google-resumable-media/tests_async/unit/test__upload.py
@@ -0,0 +1,1247 @@
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.client
+import io
+import sys
+
+import mock
+import pytest # type: ignore
+
+from google._async_resumable_media import _upload
+from google.resumable_media import common
+from google.resumable_media import _helpers as sync_helpers
+from tests.unit import test__upload as sync_test
+
+
+class TestUploadBase(object):
+ def test_constructor_defaults(self):
+ upload = _upload.UploadBase(sync_test.SIMPLE_URL)
+ assert upload.upload_url == sync_test.SIMPLE_URL
+ assert upload._headers == {}
+ assert not upload._finished
+ _check_retry_strategy(upload)
+
+ def test_constructor_explicit(self):
+ headers = {"spin": "doctors"}
+ upload = _upload.UploadBase(sync_test.SIMPLE_URL, headers=headers)
+ assert upload.upload_url == sync_test.SIMPLE_URL
+ assert upload._headers is headers
+ assert not upload._finished
+ _check_retry_strategy(upload)
+
+ def test_finished_property(self):
+ upload = _upload.UploadBase(sync_test.SIMPLE_URL)
+ # Default value of @property.
+ assert not upload.finished
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.finished = False
+
+ # Set it privately and then check the @property.
+ upload._finished = True
+ assert upload.finished
+
+ def test__process_response_bad_status(self):
+ upload = _upload.UploadBase(sync_test.SIMPLE_URL)
+ _fix_up_virtual(upload)
+
+ # Make sure **not finished** before.
+ assert not upload.finished
+ status_code = http.client.SERVICE_UNAVAILABLE
+ response = _make_response(status_code=status_code)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 4
+ assert error.args[1] == status_code
+ assert error.args[3] == http.client.OK
+ # Make sure **finished** after (even in failure).
+ assert upload.finished
+
+ def test__process_response(self):
+ upload = _upload.UploadBase(sync_test.SIMPLE_URL)
+ _fix_up_virtual(upload)
+
+ # Make sure **not finished** before.
+ assert not upload.finished
+ response = _make_response()
+ ret_val = upload._process_response(response)
+ assert ret_val is None
+ # Make sure **finished** after.
+ assert upload.finished
+
+ def test__get_status_code(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _upload.UploadBase._get_status_code(None)
+
+ exc_info.match("virtual")
+
+ def test__get_headers(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _upload.UploadBase._get_headers(None)
+
+ exc_info.match("virtual")
+
+ def test__get_body(self):
+ with pytest.raises(NotImplementedError) as exc_info:
+ _upload.UploadBase._get_body(None)
+
+ exc_info.match("virtual")
+
+
+class TestSimpleUpload(object):
+ def test__prepare_request_already_finished(self):
+ upload = _upload.SimpleUpload(sync_test.SIMPLE_URL)
+ upload._finished = True
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request(b"", None)
+
+ exc_info.match("An upload can only be used once.")
+
+ def test__prepare_request_non_bytes_data(self):
+ upload = _upload.SimpleUpload(sync_test.SIMPLE_URL)
+ assert not upload.finished
+ with pytest.raises(TypeError) as exc_info:
+ upload._prepare_request("", None)
+
+ exc_info.match("must be bytes")
+
+ def test__prepare_request(self):
+ upload = _upload.SimpleUpload(sync_test.SIMPLE_URL)
+ content_type = "image/jpeg"
+ data = b"cheetos and eetos"
+ method, url, payload, headers = upload._prepare_request(data, content_type)
+
+ assert method == "POST"
+ assert url == sync_test.SIMPLE_URL
+ assert payload == data
+ assert headers == {"content-type": content_type}
+
+ def test__prepare_request_with_headers(self):
+ headers = {"x-goog-cheetos": "spicy"}
+ upload = _upload.SimpleUpload(sync_test.SIMPLE_URL, headers=headers)
+ content_type = "image/jpeg"
+ data = b"some stuff"
+ method, url, payload, new_headers = upload._prepare_request(data, content_type)
+
+ assert method == "POST"
+ assert url == sync_test.SIMPLE_URL
+ assert payload == data
+ assert new_headers is headers
+ expected = {"content-type": content_type, "x-goog-cheetos": "spicy"}
+ assert headers == expected
+
+ def test_transmit(self):
+ upload = _upload.SimpleUpload(sync_test.SIMPLE_URL)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.transmit(None, None, None)
+
+ exc_info.match("virtual")
+
+
+class TestMultipartUpload(object):
+ def test_constructor_defaults(self):
+ upload = _upload.MultipartUpload(sync_test.MULTIPART_URL)
+ assert upload.upload_url == sync_test.MULTIPART_URL
+ assert upload._headers == {}
+ assert upload._checksum_type is None
+ assert not upload._finished
+ _check_retry_strategy(upload)
+
+ def test_constructor_explicit(self):
+ headers = {"spin": "doctors"}
+ upload = _upload.MultipartUpload(
+ sync_test.MULTIPART_URL, headers=headers, checksum="md5"
+ )
+ assert upload.upload_url == sync_test.MULTIPART_URL
+ assert upload._headers is headers
+ assert upload._checksum_type == "md5"
+ assert not upload._finished
+ _check_retry_strategy(upload)
+
+ def test__prepare_request_already_finished(self):
+ upload = _upload.MultipartUpload(sync_test.MULTIPART_URL)
+ upload._finished = True
+ with pytest.raises(ValueError):
+ upload._prepare_request(b"Hi", {}, sync_test.BASIC_CONTENT)
+
+ def test__prepare_request_non_bytes_data(self):
+ data = "Nope not bytes."
+ upload = _upload.MultipartUpload(sync_test.MULTIPART_URL)
+ with pytest.raises(TypeError):
+ upload._prepare_request(data, {}, sync_test.BASIC_CONTENT)
+
+ @mock.patch(
+ "google._async_resumable_media._upload.get_boundary", return_value=b"==3=="
+ )
+ def _prepare_request_helper(
+ self,
+ mock_get_boundary,
+ headers=None,
+ checksum=None,
+ expected_checksum=None,
+ test_overwrite=False,
+ ):
+ upload = _upload.MultipartUpload(
+ sync_test.MULTIPART_URL, headers=headers, checksum=checksum
+ )
+ data = b"Hi"
+ if test_overwrite and checksum:
+ # Deliberately set metadata that conflicts with the chosen checksum.
+ # This should be fully overwritten by the calculated checksum, so
+ # the output should not change even if this is set.
+ if checksum == "md5":
+ metadata = {"md5Hash": "ZZZZZZZZZZZZZZZZZZZZZZ=="}
+ else:
+ metadata = {"crc32c": "ZZZZZZ=="}
+ else:
+ # To simplify parsing the response, omit other test metadata if a
+ # checksum is specified.
+ metadata = {"Some": "Stuff"} if not checksum else {}
+ content_type = sync_test.BASIC_CONTENT
+ method, url, payload, new_headers = upload._prepare_request(
+ data, metadata, content_type
+ )
+
+ assert method == "POST"
+ assert url == sync_test.MULTIPART_URL
+
+ preamble = b"--==3==\r\n" + sync_test.JSON_TYPE_LINE + b"\r\n"
+
+ if checksum == "md5" and expected_checksum:
+ metadata_payload = '{{"md5Hash": "{}"}}\r\n'.format(
+ expected_checksum
+ ).encode("utf8")
+ elif checksum == "crc32c" and expected_checksum:
+ metadata_payload = '{{"crc32c": "{}"}}\r\n'.format(
+ expected_checksum
+ ).encode("utf8")
+ else:
+ metadata_payload = b'{"Some": "Stuff"}\r\n'
+ remainder = b"--==3==\r\ncontent-type: text/plain\r\n\r\nHi\r\n--==3==--"
+ expected_payload = preamble + metadata_payload + remainder
+
+ assert payload == expected_payload
+ multipart_type = b'multipart/related; boundary="==3=="'
+ mock_get_boundary.assert_called_once_with()
+
+ return new_headers, multipart_type
+
+ def test__prepare_request(self):
+ headers, multipart_type = self._prepare_request_helper()
+ assert headers == {"content-type": multipart_type}
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__prepare_request_with_checksum(self, checksum):
+ checksums = {
+ "md5": "waUpj5Oeh+j5YqXt/CBpGA==",
+ "crc32c": "ihY6wA==",
+ }
+ headers, multipart_type = self._prepare_request_helper(
+ checksum=checksum, expected_checksum=checksums[checksum]
+ )
+ assert headers == {
+ "content-type": multipart_type,
+ }
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__prepare_request_with_checksum_overwrite(self, checksum):
+ checksums = {
+ "md5": "waUpj5Oeh+j5YqXt/CBpGA==",
+ "crc32c": "ihY6wA==",
+ }
+ headers, multipart_type = self._prepare_request_helper(
+ checksum=checksum,
+ expected_checksum=checksums[checksum],
+ test_overwrite=True,
+ )
+ assert headers == {
+ "content-type": multipart_type,
+ }
+
+ def test__prepare_request_with_headers(self):
+ headers = {"best": "shirt", "worst": "hat"}
+ new_headers, multipart_type = self._prepare_request_helper(headers=headers)
+ assert new_headers is headers
+ expected_headers = {
+ "best": "shirt",
+ "content-type": multipart_type,
+ "worst": "hat",
+ }
+ assert expected_headers == headers
+
+ def test_transmit(self):
+ upload = _upload.MultipartUpload(sync_test.MULTIPART_URL)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.transmit(None, None, None, None)
+
+ exc_info.match("virtual")
+
+
+class TestResumableUpload(object):
+ def test_constructor(self):
+ chunk_size = sync_test.ONE_MB
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, chunk_size)
+ assert upload.upload_url == sync_test.RESUMABLE_URL
+ assert upload._headers == {}
+ assert not upload._finished
+ _check_retry_strategy(upload)
+ assert upload._chunk_size == chunk_size
+ assert upload._stream is None
+ assert upload._content_type is None
+ assert upload._bytes_uploaded == 0
+ assert upload._bytes_checksummed == 0
+ assert upload._checksum_object is None
+ assert upload._total_bytes is None
+ assert upload._resumable_url is None
+ assert upload._checksum_type is None
+
+ def test_constructor_bad_chunk_size(self):
+ with pytest.raises(ValueError):
+ _upload.ResumableUpload(sync_test.RESUMABLE_URL, 1)
+
+ def test_invalid_property(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ # Default value of @property.
+ assert not upload.invalid
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.invalid = False
+
+ # Set it privately and then check the @property.
+ upload._invalid = True
+ assert upload.invalid
+
+ def test_chunk_size_property(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ # Default value of @property.
+ assert upload.chunk_size == sync_test.ONE_MB
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.chunk_size = 17
+
+ # Set it privately and then check the @property.
+ new_size = 102
+ upload._chunk_size = new_size
+ assert upload.chunk_size == new_size
+
+ def test_resumable_url_property(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ # Default value of @property.
+ assert upload.resumable_url is None
+
+ # Make sure we cannot set it on public @property.
+ new_url = "http://test.invalid?upload_id=not-none"
+ with pytest.raises(AttributeError):
+ upload.resumable_url = new_url
+
+ # Set it privately and then check the @property.
+ upload._resumable_url = new_url
+ assert upload.resumable_url == new_url
+
+ def test_bytes_uploaded_property(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ # Default value of @property.
+ assert upload.bytes_uploaded == 0
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.bytes_uploaded = 1024
+
+ # Set it privately and then check the @property.
+ upload._bytes_uploaded = 128
+ assert upload.bytes_uploaded == 128
+
+ def test_total_bytes_property(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ # Default value of @property.
+ assert upload.total_bytes is None
+
+ # Make sure we cannot set it on public @property.
+ with pytest.raises(AttributeError):
+ upload.total_bytes = 65536
+
+ # Set it privately and then check the @property.
+ upload._total_bytes = 8192
+ assert upload.total_bytes == 8192
+
+ def _prepare_initiate_request_helper(self, upload_headers=None, **method_kwargs):
+ data = b"some really big big data."
+ stream = io.BytesIO(data)
+ metadata = {"name": "big-data-file.txt"}
+
+ upload = _upload.ResumableUpload(
+ sync_test.RESUMABLE_URL, sync_test.ONE_MB, headers=upload_headers
+ )
+ orig_headers = upload._headers.copy()
+ # Check ``upload``-s state before.
+ assert upload._stream is None
+ assert upload._content_type is None
+ assert upload._total_bytes is None
+ # Call the method and check the output.
+ method, url, payload, headers = upload._prepare_initiate_request(
+ stream, metadata, sync_test.BASIC_CONTENT, **method_kwargs
+ )
+ assert payload == b'{"name": "big-data-file.txt"}'
+ # Make sure the ``upload``-s state was updated.
+ assert upload._stream == stream
+ assert upload._content_type == sync_test.BASIC_CONTENT
+ if method_kwargs == {"stream_final": False}:
+ assert upload._total_bytes is None
+ else:
+ assert upload._total_bytes == len(data)
+ # Make sure headers are untouched.
+ assert headers is not upload._headers
+ assert upload._headers == orig_headers
+ assert method == "POST"
+ assert url == upload.upload_url
+ # Make sure the stream is still at the beginning.
+ assert stream.tell() == 0
+
+ return data, headers
+
+ def test__prepare_initiate_request(self):
+ data, headers = self._prepare_initiate_request_helper()
+ expected_headers = {
+ "content-type": sync_test.JSON_TYPE,
+ "x-upload-content-length": "{:d}".format(len(data)),
+ "x-upload-content-type": sync_test.BASIC_CONTENT,
+ }
+ assert headers == expected_headers
+
+ def test__prepare_initiate_request_with_headers(self):
+ headers = {"caviar": "beluga", "top": "quark"}
+ data, new_headers = self._prepare_initiate_request_helper(
+ upload_headers=headers
+ )
+ expected_headers = {
+ "caviar": "beluga",
+ "content-type": sync_test.JSON_TYPE,
+ "top": "quark",
+ "x-upload-content-length": "{:d}".format(len(data)),
+ "x-upload-content-type": sync_test.BASIC_CONTENT,
+ }
+ assert new_headers == expected_headers
+
+ def test__prepare_initiate_request_known_size(self):
+ total_bytes = 25
+ data, headers = self._prepare_initiate_request_helper(total_bytes=total_bytes)
+ assert len(data) == total_bytes
+ expected_headers = {
+ "content-type": "application/json; charset=UTF-8",
+ "x-upload-content-length": "{:d}".format(total_bytes),
+ "x-upload-content-type": sync_test.BASIC_CONTENT,
+ }
+ assert headers == expected_headers
+
+ def test__prepare_initiate_request_unknown_size(self):
+ _, headers = self._prepare_initiate_request_helper(stream_final=False)
+ expected_headers = {
+ "content-type": "application/json; charset=UTF-8",
+ "x-upload-content-type": sync_test.BASIC_CONTENT,
+ }
+ assert headers == expected_headers
+
+ def test__prepare_initiate_request_already_initiated(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ # Fake that the upload has been started.
+ upload._resumable_url = "http://test.invalid?upload_id=definitely-started"
+
+ with pytest.raises(ValueError):
+ upload._prepare_initiate_request(io.BytesIO(), {}, sync_test.BASIC_CONTENT)
+
+ def test__prepare_initiate_request_bad_stream_position(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+
+ stream = io.BytesIO(b"data")
+ stream.seek(1)
+ with pytest.raises(ValueError):
+ upload._prepare_initiate_request(stream, {}, sync_test.BASIC_CONTENT)
+
+ # Also test a bad object (i.e. non-stream)
+ with pytest.raises(AttributeError):
+ upload._prepare_initiate_request(None, {}, sync_test.BASIC_CONTENT)
+
+ def test__process_initiate_response_non_200(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ response = _make_response(403)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_initiate_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 4
+ assert error.args[1] == 403
+ assert error.args[3] == 200
+
+ def test__process_initiate_response(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ headers = {"location": "http://test.invalid?upload_id=kmfeij3234"}
+ response = _make_response(headers=headers)
+ # Check resumable_url before.
+ assert upload._resumable_url is None
+ # Process the actual headers.
+ ret_val = upload._process_initiate_response(response)
+ assert ret_val is None
+ # Check resumable_url after.
+ assert upload._resumable_url == headers["location"]
+
+ def test_initiate(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.initiate(None, None, {}, sync_test.BASIC_CONTENT)
+
+ exc_info.match("virtual")
+
+ def test__prepare_request_already_finished(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ assert not upload.invalid
+ upload._finished = True
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request()
+
+ assert exc_info.value.args == ("Upload has finished.",)
+
+ def test__prepare_request_invalid(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ assert not upload.finished
+ upload._invalid = True
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request()
+
+ assert exc_info.match("invalid state")
+ assert exc_info.match("recover()")
+
+ def test__prepare_request_not_initiated(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ assert not upload.finished
+ assert not upload.invalid
+ assert upload._resumable_url is None
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request()
+
+ assert exc_info.match("upload has not been initiated")
+ assert exc_info.match("initiate()")
+
+ def test__prepare_request_invalid_stream_state(self):
+ stream = io.BytesIO(b"some data here")
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ upload._stream = stream
+ upload._resumable_url = "http://test.invalid?upload_id=not-none"
+ # Make stream.tell() disagree with bytes_uploaded.
+ upload._bytes_uploaded = 5
+ assert upload.bytes_uploaded != stream.tell()
+ with pytest.raises(ValueError) as exc_info:
+ upload._prepare_request()
+
+ assert exc_info.match("Bytes stream is in unexpected state.")
+
+ @staticmethod
+ def _upload_in_flight(data, headers=None, checksum=None):
+ upload = _upload.ResumableUpload(
+ sync_test.RESUMABLE_URL,
+ sync_test.ONE_MB,
+ headers=headers,
+ checksum=checksum,
+ )
+ upload._stream = io.BytesIO(data)
+ upload._content_type = sync_test.BASIC_CONTENT
+ upload._total_bytes = len(data)
+ upload._resumable_url = "http://test.invalid?upload_id=not-none"
+ return upload
+
+ def _prepare_request_helper(self, headers=None):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, headers=headers)
+ method, url, payload, new_headers = upload._prepare_request()
+ # Check the response values.
+ assert method == "PUT"
+ assert url == upload.resumable_url
+ assert payload == data
+ # Make sure headers are **NOT** updated
+ assert upload._headers != new_headers
+
+ return new_headers
+
+ def test__prepare_request_success(self):
+ headers = self._prepare_request_helper()
+ expected_headers = {
+ "content-range": "bytes 0-32/33",
+ "content-type": sync_test.BASIC_CONTENT,
+ }
+ assert headers == expected_headers
+
+ def test__prepare_request_success_with_headers(self):
+ headers = {"cannot": "touch this"}
+ new_headers = self._prepare_request_helper(headers)
+ assert new_headers is not headers
+ expected_headers = {
+ "content-range": "bytes 0-32/33",
+ "content-type": sync_test.BASIC_CONTENT,
+ }
+ assert new_headers == expected_headers
+ # Make sure the ``_headers`` are not incorporated.
+ assert "cannot" not in new_headers
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__prepare_request_with_checksum(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ upload._prepare_request()
+ assert upload._checksum_object is not None
+
+ checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ checksum_digest = sync_helpers.prepare_checksum_digest(
+ upload._checksum_object.digest()
+ )
+ assert checksum_digest == checksums[checksum]
+ assert upload._bytes_checksummed == len(data)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__update_checksum(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 8
+
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 16
+
+ # Continue to the end.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ checksum_digest = sync_helpers.prepare_checksum_digest(
+ upload._checksum_object.digest()
+ )
+ assert checksum_digest == checksums[checksum]
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ def test__update_checksum_rewind(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 8
+ checksum_checkpoint = upload._checksum_object.digest()
+
+ # Rewind to the beginning.
+ upload._stream.seek(0)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 8
+ assert upload._checksum_object.digest() == checksum_checkpoint
+
+ # Rewind but not to the beginning.
+ upload._stream.seek(4)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == 12
+
+ # Continue to the end.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ checksums = {"md5": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ checksum_digest = sync_helpers.prepare_checksum_digest(
+ upload._checksum_object.digest()
+ )
+ assert checksum_digest == checksums[checksum]
+
+ def test__update_checksum_none(self):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=None)
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ upload._update_checksum(start_byte, payload)
+ assert upload._checksum_object is None
+
+ def test__update_checksum_invalid(self):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum="invalid")
+ start_byte, payload, _ = _upload.get_next_chunk(upload._stream, 8, len(data))
+ with pytest.raises(ValueError):
+ upload._update_checksum(start_byte, payload)
+
+ def test__make_invalid(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ assert not upload.invalid
+ upload._make_invalid()
+ assert upload.invalid
+
+ @pytest.mark.asyncio
+ async def test__process_resumable_response_bad_status(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ # Make sure the upload is valid before the failure.
+ assert not upload.invalid
+ response = _make_response(status_code=http.client.NOT_FOUND)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await upload._process_resumable_response(response, None)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 5
+ assert error.args[1] == response.status_code
+ assert error.args[3] == http.client.OK
+ assert error.args[4] == http.client.PERMANENT_REDIRECT
+ # Make sure the upload is invalid after the failure.
+ assert upload.invalid
+
+ @pytest.mark.asyncio
+ async def test__process_resumable_response_success(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ # Check / set status before.
+ assert upload._bytes_uploaded == 0
+ upload._bytes_uploaded = 20
+ assert not upload._finished
+
+ # Set the response body.
+ bytes_sent = 158
+ total_bytes = upload._bytes_uploaded + bytes_sent
+ response_body = '{{"size": "{:d}"}}'.format(total_bytes)
+ response_body = response_body.encode("utf-8")
+ response = mock.Mock(
+ content=response_body,
+ status_code=http.client.OK,
+ spec=["content", "status_code"],
+ )
+ ret_val = await upload._process_resumable_response(response, bytes_sent)
+ assert ret_val is None
+ # Check status after.
+ assert upload._bytes_uploaded == total_bytes
+ assert upload._finished
+
+ @pytest.mark.asyncio
+ async def test__process_resumable_response_partial_no_range(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ response = _make_response(status_code=http.client.PERMANENT_REDIRECT)
+ # Make sure the upload is valid before the failure.
+ assert not upload.invalid
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await upload._process_resumable_response(response, None)
+ # Make sure the upload is invalid after the failure.
+ assert upload.invalid
+
+ # Check the error response.
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 2
+ assert error.args[1] == "range"
+
+ @pytest.mark.asyncio
+ async def test__process_resumable_response_partial_bad_range(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ # Make sure the upload is valid before the failure.
+ assert not upload.invalid
+ headers = {"range": "nights 1-81"}
+ response = _make_response(
+ status_code=http.client.PERMANENT_REDIRECT, headers=headers
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await upload._process_resumable_response(response, 81)
+
+ # Check the error response.
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == headers["range"]
+ # Make sure the upload is invalid after the failure.
+ assert upload.invalid
+
+ @pytest.mark.asyncio
+ async def test__process_resumable_response_partial(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ # Check status before.
+ assert upload._bytes_uploaded == 0
+ headers = {"range": "bytes=0-171"}
+ response = _make_response(
+ status_code=http.client.PERMANENT_REDIRECT, headers=headers
+ )
+ ret_val = await upload._process_resumable_response(response, 172)
+ assert ret_val is None
+ # Check status after.
+ assert upload._bytes_uploaded == 172
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test__validate_checksum_success(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ _fix_up_virtual(upload)
+ # Go ahead and process the entire data in one go for this test.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ # This is only used by _validate_checksum for fetching metadata and
+ # logging.
+ metadata = {"md5Hash": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ response = _make_response(headers=metadata)
+ upload._finished = True
+
+ assert upload._checksum_object is not None
+ # Test passes if it does not raise an error (no assert needed)
+ await upload._validate_checksum(response)
+
+ @pytest.mark.asyncio
+ async def test__validate_checksum_none(self):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(b"test", checksum=None)
+ _fix_up_virtual(upload)
+ # Go ahead and process the entire data in one go for this test.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+
+ # This is only used by _validate_checksum for fetching metadata and
+ # logging.
+ metadata = {"md5Hash": "GRvfKbqr5klAOwLkxgIf8w==", "crc32c": "Qg8thA=="}
+ response = _make_response(headers=metadata)
+ upload._finished = True
+
+ assert upload._checksum_object is None
+ assert upload._bytes_checksummed == 0
+ # Test passes if it does not raise an error (no assert needed)
+ await upload._validate_checksum(response)
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test__validate_checksum_header_no_match(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ _fix_up_virtual(upload)
+ # Go ahead and process the entire data in one go for this test.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ # For this test, each checksum option will be provided with a valid but
+ # mismatching remote checksum type.
+ if checksum == "crc32c":
+ metadata = {"md5Hash": "GRvfKbqr5klAOwLkxgIf8w=="}
+ else:
+ metadata = {"crc32c": "Qg8thA=="}
+ # This is only used by _validate_checksum for fetching headers and
+ # logging, so it doesn't need to be fleshed out with a response body.
+ response = _make_response(headers=metadata)
+ upload._finished = True
+
+ assert upload._checksum_object is not None
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ await upload._validate_checksum(response)
+
+ error = exc_info.value
+ assert error.response is response
+ message = error.args[0]
+ metadata_key = sync_helpers._get_metadata_key(checksum)
+ assert (
+ message
+ == _upload._UPLOAD_METADATA_NO_APPROPRIATE_CHECKSUM_MESSAGE.format(
+ metadata_key
+ )
+ )
+
+ @pytest.mark.parametrize("checksum", ["md5", "crc32c"])
+ @pytest.mark.asyncio
+ async def test__validate_checksum_mismatch(self, checksum):
+ data = b"All of the data goes in a stream."
+ upload = self._upload_in_flight(data, checksum=checksum)
+ _fix_up_virtual(upload)
+ # Go ahead and process the entire data in one go for this test.
+ start_byte, payload, _ = _upload.get_next_chunk(
+ upload._stream, len(data), len(data)
+ )
+ upload._update_checksum(start_byte, payload)
+ assert upload._bytes_checksummed == len(data)
+
+ metadata = {
+ "md5Hash": "ZZZZZZZZZZZZZZZZZZZZZZ==",
+ "crc32c": "ZZZZZZ==",
+ }
+ # This is only used by _validate_checksum for fetching headers and
+ # logging, so it doesn't need to be fleshed out with a response body.
+ response = _make_response(headers=metadata)
+ upload._finished = True
+
+ assert upload._checksum_object is not None
+ # Test passes if it does not raise an error (no assert needed)
+ with pytest.raises(common.DataCorruption) as exc_info:
+ await upload._validate_checksum(response)
+
+ error = exc_info.value
+ assert error.response is response
+ message = error.args[0]
+ correct_checksums = {"crc32c": "Qg8thA==", "md5": "GRvfKbqr5klAOwLkxgIf8w=="}
+ metadata_key = sync_helpers._get_metadata_key(checksum)
+ assert message == _upload._UPLOAD_CHECKSUM_MISMATCH_MESSAGE.format(
+ checksum.upper(), correct_checksums[checksum], metadata[metadata_key]
+ )
+
+ def test_transmit_next_chunk(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.transmit_next_chunk(None)
+
+ exc_info.match("virtual")
+
+ def test__prepare_recover_request_not_invalid(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ assert not upload.invalid
+
+ with pytest.raises(ValueError):
+ upload._prepare_recover_request()
+
+ def test__prepare_recover_request(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ upload._invalid = True
+
+ method, url, payload, headers = upload._prepare_recover_request()
+ assert method == "PUT"
+ assert url == upload.resumable_url
+ assert payload is None
+ assert headers == {"content-range": "bytes */*"}
+ # Make sure headers are untouched.
+ assert upload._headers == {}
+
+ def test__prepare_recover_request_with_headers(self):
+ headers = {"lake": "ocean"}
+ upload = _upload.ResumableUpload(
+ sync_test.RESUMABLE_URL, sync_test.ONE_MB, headers=headers
+ )
+ upload._invalid = True
+
+ method, url, payload, new_headers = upload._prepare_recover_request()
+ assert method == "PUT"
+ assert url == upload.resumable_url
+ assert payload is None
+ assert new_headers == {"content-range": "bytes */*"}
+ # Make sure the ``_headers`` are not incorporated.
+ assert "lake" not in new_headers
+ # Make sure headers are untouched.
+ assert upload._headers == {"lake": "ocean"}
+
+ def test__process_recover_response_bad_status(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ upload._invalid = True
+
+ response = _make_response(status_code=http.client.BAD_REQUEST)
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_recover_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 4
+ assert error.args[1] == response.status_code
+ assert error.args[3] == http.client.PERMANENT_REDIRECT
+ # Make sure still invalid.
+ assert upload.invalid
+
+ def test__process_recover_response_no_range(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ upload._invalid = True
+ upload._stream = mock.Mock(spec=["seek"])
+ upload._bytes_uploaded = mock.sentinel.not_zero
+ assert upload.bytes_uploaded != 0
+
+ response = _make_response(status_code=http.client.PERMANENT_REDIRECT)
+ ret_val = upload._process_recover_response(response)
+ assert ret_val is None
+ # Check the state of ``upload`` after.
+ assert upload.bytes_uploaded == 0
+ assert not upload.invalid
+ upload._stream.seek.assert_called_once_with(0)
+
+ def test__process_recover_response_bad_range(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ upload._invalid = True
+ upload._stream = mock.Mock(spec=["seek"])
+ upload._bytes_uploaded = mock.sentinel.not_zero
+
+ headers = {"range": "bites=9-11"}
+ response = _make_response(
+ status_code=http.client.PERMANENT_REDIRECT, headers=headers
+ )
+ with pytest.raises(common.InvalidResponse) as exc_info:
+ upload._process_recover_response(response)
+
+ error = exc_info.value
+ assert error.response is response
+ assert len(error.args) == 3
+ assert error.args[1] == headers["range"]
+ # Check the state of ``upload`` after (untouched).
+ assert upload.bytes_uploaded is mock.sentinel.not_zero
+ assert upload.invalid
+ upload._stream.seek.assert_not_called()
+
+ def test__process_recover_response_with_range(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ _fix_up_virtual(upload)
+
+ upload._invalid = True
+ upload._stream = mock.Mock(spec=["seek"])
+ upload._bytes_uploaded = mock.sentinel.not_zero
+ assert upload.bytes_uploaded != 0
+
+ end = 11
+ headers = {"range": "bytes=0-{:d}".format(end)}
+ response = _make_response(
+ status_code=http.client.PERMANENT_REDIRECT, headers=headers
+ )
+ ret_val = upload._process_recover_response(response)
+ assert ret_val is None
+ # Check the state of ``upload`` after.
+ assert upload.bytes_uploaded == end + 1
+ assert not upload.invalid
+ upload._stream.seek.assert_called_once_with(end + 1)
+
+ def test_recover(self):
+ upload = _upload.ResumableUpload(sync_test.RESUMABLE_URL, sync_test.ONE_MB)
+ with pytest.raises(NotImplementedError) as exc_info:
+ upload.recover(None)
+
+ exc_info.match("virtual")
+
+
+@mock.patch("random.randrange", return_value=1234567890123456789)
+def test_get_boundary(mock_rand):
+ result = _upload.get_boundary()
+ assert result == b"===============1234567890123456789=="
+ mock_rand.assert_called_once_with(sys.maxsize)
+
+
+class Test_construct_multipart_request(object):
+ @mock.patch(
+ "google._async_resumable_media._upload.get_boundary", return_value=b"==1=="
+ )
+ def test_binary(self, mock_get_boundary):
+ data = b"By nary day tuh"
+ metadata = {"name": "hi-file.bin"}
+ content_type = "application/octet-stream"
+ payload, multipart_boundary = _upload.construct_multipart_request(
+ data, metadata, content_type
+ )
+
+ assert multipart_boundary == mock_get_boundary.return_value
+ expected_payload = (
+ b"--==1==\r\n" + sync_test.JSON_TYPE_LINE + b"\r\n"
+ b'{"name": "hi-file.bin"}\r\n'
+ b"--==1==\r\n"
+ b"content-type: application/octet-stream\r\n"
+ b"\r\n"
+ b"By nary day tuh\r\n"
+ b"--==1==--"
+ )
+ assert payload == expected_payload
+ mock_get_boundary.assert_called_once_with()
+
+ @mock.patch(
+ "google._async_resumable_media._upload.get_boundary", return_value=b"==2=="
+ )
+ def test_unicode(self, mock_get_boundary):
+ data_unicode = "\N{SNOWMAN}"
+ # construct_multipart_request( ASSUMES callers pass bytes.
+ data = data_unicode.encode("utf-8")
+ metadata = {"name": "snowman.txt"}
+ content_type = sync_test.BASIC_CONTENT
+ payload, multipart_boundary = _upload.construct_multipart_request(
+ data, metadata, content_type
+ )
+
+ assert multipart_boundary == mock_get_boundary.return_value
+ expected_payload = (
+ b"--==2==\r\n" + sync_test.JSON_TYPE_LINE + b"\r\n"
+ b'{"name": "snowman.txt"}\r\n'
+ b"--==2==\r\n"
+ b"content-type: text/plain\r\n"
+ b"\r\n"
+ b"\xe2\x98\x83\r\n"
+ b"--==2==--"
+ )
+ assert payload == expected_payload
+ mock_get_boundary.assert_called_once_with()
+
+
+def test_get_total_bytes():
+ data = b"some data"
+ stream = io.BytesIO(data)
+ # Check position before function call.
+ assert stream.tell() == 0
+ assert _upload.get_total_bytes(stream) == len(data)
+ # Check position after function call.
+ assert stream.tell() == 0
+
+ # Make sure this works just as well when not at beginning.
+ curr_pos = 3
+ stream.seek(curr_pos)
+ assert _upload.get_total_bytes(stream) == len(data)
+ # Check position after function call.
+ assert stream.tell() == curr_pos
+
+
+class Test_get_next_chunk(object):
+ def test_exhausted_known_size(self):
+ data = b"the end"
+ stream = io.BytesIO(data)
+ stream.seek(len(data))
+ with pytest.raises(ValueError) as exc_info:
+ _upload.get_next_chunk(stream, 1, len(data))
+
+ exc_info.match("Stream is already exhausted. There is no content remaining.")
+
+ def test_exhausted_known_size_zero(self):
+ stream = io.BytesIO(b"")
+ answer = _upload.get_next_chunk(stream, 1, 0)
+ assert answer == (0, b"", "bytes */0")
+
+ def test_exhausted_known_size_zero_nonempty(self):
+ stream = io.BytesIO(b"not empty WAT!")
+ with pytest.raises(ValueError) as exc_info:
+ _upload.get_next_chunk(stream, 1, 0)
+ exc_info.match("Stream specified as empty, but produced non-empty content.")
+
+ def test_success_known_size_lt_stream_size(self):
+ data = b"0123456789"
+ stream = io.BytesIO(data)
+ chunk_size = 3
+ total_bytes = len(data) - 2
+
+ # Splits into 3 chunks: 012, 345, 67
+ result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+
+ assert result0 == (0, b"012", "bytes 0-2/8")
+ assert result1 == (3, b"345", "bytes 3-5/8")
+ assert result2 == (6, b"67", "bytes 6-7/8")
+
+ def test_success_known_size(self):
+ data = b"0123456789"
+ stream = io.BytesIO(data)
+ total_bytes = len(data)
+ chunk_size = 3
+ # Splits into 4 chunks: 012, 345, 678, 9
+ result0 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result1 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result2 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ result3 = _upload.get_next_chunk(stream, chunk_size, total_bytes)
+ assert result0 == (0, b"012", "bytes 0-2/10")
+ assert result1 == (3, b"345", "bytes 3-5/10")
+ assert result2 == (6, b"678", "bytes 6-8/10")
+ assert result3 == (9, b"9", "bytes 9-9/10")
+ assert stream.tell() == total_bytes
+
+ def test_success_unknown_size(self):
+ data = b"abcdefghij"
+ stream = io.BytesIO(data)
+ chunk_size = 6
+ # Splits into 4 chunks: abcdef, ghij
+ result0 = _upload.get_next_chunk(stream, chunk_size, None)
+ result1 = _upload.get_next_chunk(stream, chunk_size, None)
+ assert result0 == (0, b"abcdef", "bytes 0-5/*")
+ assert result1 == (chunk_size, b"ghij", "bytes 6-9/10")
+ assert stream.tell() == len(data)
+
+ # Do the same when the chunk size evenly divides len(data)
+ stream.seek(0)
+ chunk_size = len(data)
+ # Splits into 2 chunks: `data` and empty string
+ result0 = _upload.get_next_chunk(stream, chunk_size, None)
+ result1 = _upload.get_next_chunk(stream, chunk_size, None)
+ assert result0 == (0, data, "bytes 0-9/*")
+ assert result1 == (len(data), b"", "bytes */10")
+ assert stream.tell() == len(data)
+
+
+class Test_get_content_range(object):
+ def test_known_size(self):
+ result = _upload.get_content_range(5, 10, 40)
+ assert result == "bytes 5-10/40"
+
+ def test_unknown_size(self):
+ result = _upload.get_content_range(1000, 10000, None)
+ assert result == "bytes 1000-10000/*"
+
+
+def _make_response(status_code=http.client.OK, headers=None):
+ headers = headers or {}
+
+ response = mock.AsyncMock(
+ _headers=headers,
+ headers=headers,
+ status_code=status_code,
+ spec=["_headers", "headers", "status_code", "json"],
+ )
+ response.json = mock.AsyncMock(spec=["__call__"], return_value=headers)
+
+ return response
+
+
+def _get_status_code(response):
+ return response.status_code
+
+
+def _get_headers(response):
+ return response._headers
+
+
+def _fix_up_virtual(upload):
+ upload._get_status_code = _get_status_code
+ upload._get_headers = _get_headers
+
+
+def _check_retry_strategy(upload):
+ retry_strategy = upload._retry_strategy
+ assert isinstance(retry_strategy, common.RetryStrategy)
+ assert retry_strategy.max_sleep == common.MAX_SLEEP
+ assert retry_strategy.max_cumulative_retry == common.MAX_CUMULATIVE_RETRY
+ assert retry_strategy.max_retries is None
diff --git a/scripts/split_repo_migration/single-library.git-migrate-history.sh b/scripts/split_repo_migration/single-library.git-migrate-history.sh
index 9f926e1d030e..650abac5ca26 100755
--- a/scripts/split_repo_migration/single-library.git-migrate-history.sh
+++ b/scripts/split_repo_migration/single-library.git-migrate-history.sh
@@ -70,7 +70,7 @@ echo "Created working directory: ${WORKDIR}"
pushd "${WORKDIR}" # cd into workdir
echo "Cloning source repository: ${SOURCE_REPO}"
-git clone --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo
+git clone --recurse-submodules --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo
pushd source-repo
@@ -124,7 +124,7 @@ git filter-branch \
--force \
--prune-empty \
--tree-filter \
- "git submodule update --init --recursive; find . -mindepth 2 -name .git -exec rm -rf {} +; shopt -s dotglob; mkdir -p ${WORKDIR}/migrated-source; mv * ${WORKDIR}/migrated-source; mkdir -p ${TARGET_PATH}; { mv ${WORKDIR}/migrated-source/* ${TARGET_PATH} || echo 'No files to move' ; }"
+ "git submodule update --init --recursive; find . -mindepth 2 -name .git -exec rm -rf {} +; git submodule update --init --recursive; find . -mindepth 2 -name .git -exec rm -rf {} +; shopt -s dotglob; mkdir -p ${WORKDIR}/migrated-source; mv * ${WORKDIR}/migrated-source; mkdir -p ${TARGET_PATH}; { mv ${WORKDIR}/migrated-source/* ${TARGET_PATH} || echo 'No files to move' ; }"
# back to workdir
popd
@@ -142,7 +142,7 @@ echo "Success"
popd # back to workdir
# Do a diff between source code split repo and migrated code.
-git clone --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo-validation # Not ideal to clone again.
+git clone --recurse-submodules --recurse-submodules "git@github.com:${SOURCE_REPO}.git" source-repo-validation # Not ideal to clone again.
find source-repo-validation -name .git -exec rm -rf {} + # That folder is not needed for validation.
DIFF_FILE="${WORKDIR}/diff.txt"