[Python-checkins] [3.8] gh-98517: Fix buffer overflows in _sha3 module (GH-98519) (#98527)

[ambv]

https://github.com/python/cpython/commit/948c6794711458fd148a3fa62296cadeeb2ed631
commit: 948c6794711458fd148a3fa62296cadeeb2ed631
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: ambv <lukasz at langa.pl>
date: 2022-10-28T12:07:50+02:00
summary:

[3.8] gh-98517: Fix buffer overflows in _sha3 module (GH-98519) (#98527)

This is a port of the applicable part of XKCP's fix [1] for
CVE-2022-37454 and avoids the segmentation fault and the infinite
loop in the test cases published in [2].

[1]: https://github.com/XKCP/XKCP/commit/fdc6fef075f4e81d6b1bc38364248975e08e340a
[2]: https://mouha.be/sha-3-buffer-overflow/

Regression test added by: Gregory P. Smith [Google LLC] <greg at krypto.org>
(cherry picked from commit 0e4e058602d93b88256ff90bbef501ba20be9dd3)

Co-authored-by: Theo Buehler <botovq at users.noreply.github.com>

files:
A Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst
M Lib/test/test_hashlib.py
M Modules/_sha3/kcp/KeccakSponge.inc

diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 8b53d23ef525..e6cec4e306e5 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -434,6 +434,15 @@ def test_case_md5_huge(self, size):
     def test_case_md5_uintmax(self, size):
         self.check('md5', b'A'*size, '28138d306ff1b8281f1a9067e1a1a2b3')
 
+    @unittest.skipIf(sys.maxsize < _4G - 1, 'test cannot run on 32-bit systems')
+    @bigmemtest(size=_4G - 1, memuse=1, dry_run=False)
+    def test_sha3_update_overflow(self, size):
+        """Regression test for gh-98517 CVE-2022-37454."""
+        h = hashlib.sha3_224()
+        h.update(b'\x01')
+        h.update(b'\x01'*0xffff_ffff)
+        self.assertEqual(h.hexdigest(), '80762e8ce6700f114fec0f621fd97c4b9c00147fa052215294cceeed')
+
     # use the three examples from Federal Information Processing Standards
     # Publication 180-1, Secure Hash Standard,  1995 April 17
     # http://www.itl.nist.gov/div897/pubs/fip180-1.htm
diff --git a/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst
new file mode 100644
index 000000000000..2d23a6ad93c7
--- /dev/null
+++ b/Misc/NEWS.d/next/Security/2022-10-21-13-31-47.gh-issue-98517.SXXGfV.rst
@@ -0,0 +1 @@
+Port XKCP's fix for the buffer overflows in SHA-3 (CVE-2022-37454).
diff --git a/Modules/_sha3/kcp/KeccakSponge.inc b/Modules/_sha3/kcp/KeccakSponge.inc
index e10739deafa8..cf92e4db4d36 100644
--- a/Modules/_sha3/kcp/KeccakSponge.inc
+++ b/Modules/_sha3/kcp/KeccakSponge.inc
@@ -171,7 +171,7 @@ int SpongeAbsorb(SpongeInstance *instance, const unsigned char *data, size_t dat
     i = 0;
     curData = data;
     while(i < dataByteLen) {
-        if ((instance->byteIOIndex == 0) && (dataByteLen >= (i + rateInBytes))) {
+        if ((instance->byteIOIndex == 0) && (dataByteLen-i >= rateInBytes)) {
 #ifdef SnP_FastLoop_Absorb
             /* processing full blocks first */
 
@@ -199,10 +199,10 @@ int SpongeAbsorb(SpongeInstance *instance, const unsigned char *data, size_t dat
         }
         else {
             /* normal lane: using the message queue */
-
-            partialBlock = (unsigned int)(dataByteLen - i);
-            if (partialBlock+instance->byteIOIndex > rateInBytes)
+            if (dataByteLen-i > rateInBytes-instance->byteIOIndex)
                 partialBlock = rateInBytes-instance->byteIOIndex;
+            else
+                partialBlock = (unsigned int)(dataByteLen - i);
             #ifdef KeccakReference
             displayBytes(1, "Block to be absorbed (part)", curData, partialBlock);
             #endif
@@ -281,7 +281,7 @@ int SpongeSqueeze(SpongeInstance *instance, unsigned char *data, size_t dataByte
     i = 0;
     curData = data;
     while(i < dataByteLen) {
-        if ((instance->byteIOIndex == rateInBytes) && (dataByteLen >= (i + rateInBytes))) {
+        if ((instance->byteIOIndex == rateInBytes) && (dataByteLen-i >= rateInBytes)) {
             for(j=dataByteLen-i; j>=rateInBytes; j-=rateInBytes) {
                 SnP_Permute(instance->state);
                 SnP_ExtractBytes(instance->state, curData, 0, rateInBytes);
@@ -299,9 +299,10 @@ int SpongeSqueeze(SpongeInstance *instance, unsigned char *data, size_t dataByte
                 SnP_Permute(instance->state);
                 instance->byteIOIndex = 0;
             }
-            partialBlock = (unsigned int)(dataByteLen - i);
-            if (partialBlock+instance->byteIOIndex > rateInBytes)
+            if (dataByteLen-i > rateInBytes-instance->byteIOIndex)
                 partialBlock = rateInBytes-instance->byteIOIndex;
+            else
+                partialBlock = (unsigned int)(dataByteLen - i);
             i += partialBlock;
 
             SnP_ExtractBytes(instance->state, curData, instance->byteIOIndex, partialBlock);