Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-114250: Fetch metadata for pip and its vendored dependencies from PyPI (#114450)

This commit is contained in:
Seth Michael Larson 2024-01-26 03:48:13 -06:00 committed by GitHub
parent 456e274578
commit 582d95e8bb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 842 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

624
Misc/sbom.spdx.json generated
View File

@ -1695,6 +1695,510 @@
"primaryPackagePurpose": "SOURCE",
"versionInfo": "2.5.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-cachecontrol",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "95dedbec849f46dda3137866dc28b9d133fc9af55f5b805ab1291833e4457aa4"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/1d/e3/a22348e6226dcd585d5a4b5f0175b3a16dabfd3912cbeb02f321d00e56c7/cachecontrol-0.13.1-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/cachecontrol@0.13.1",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "cachecontrol",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "0.13.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-colorama",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/colorama@0.4.6",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "colorama",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "0.4.6"
},
{
"SPDXID": "SPDXRef-PACKAGE-distlib",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/76/cb/6bbd2b10170ed991cf64e8c8b85e01f2fb38f95d1bc77617569e0b0b26ac/distlib-0.3.6-py2.py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/distlib@0.3.6",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "distlib",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "0.3.6"
},
{
"SPDXID": "SPDXRef-PACKAGE-distro",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "99522ca3e365cac527b44bde033f64c6945d90eb9f769703caaec52b09bbd3ff"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/f4/2c/c90a3adaf0ddb70afe193f5ebfb539612af57cffe677c3126be533df3098/distro-1.8.0-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/distro@1.8.0",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "distro",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "1.8.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-msgpack",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/9f/4a/36d936e54cf71e23ad276564465f6a54fb129e3d61520b76e13e0bb29167/msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/msgpack@1.0.5",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "msgpack",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "1.0.5"
},
{
"SPDXID": "SPDXRef-PACKAGE-packaging",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/05/8e/8de486cbd03baba4deef4142bd643a3e7bbe954a784dc1bb17142572d127/packaging-21.3-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/packaging@21.3",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "packaging",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "21.3"
},
{
"SPDXID": "SPDXRef-PACKAGE-platformdirs",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/9e/d8/563a9fc17153c588c8c2042d2f0f84a89057cdb1c30270f589c88b42d62c/platformdirs-3.8.1-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/platformdirs@3.8.1",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "platformdirs",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "3.8.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-pyparsing",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "d554a96d1a7d3ddaf7183104485bc19fd80543ad6ac5bdb6426719d766fb06c1"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/a4/24/6ae4c9c45cf99d96b06b5d99e25526c060303171fb0aea9da2bfd7dbde93/pyparsing-3.1.0-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/pyparsing@3.1.0",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "pyparsing",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "3.1.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-pyproject-hooks",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/d5/ea/9ae603de7fbb3df820b23a70f6aff92bf8c7770043254ad8d2dc9d6bcba4/pyproject_hooks-1.0.0-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/pyproject-hooks@1.0.0",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "pyproject-hooks",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "1.0.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-requests",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/requests@2.31.0",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "requests",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "2.31.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-certifi",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/4c/dd/2234eab22353ffc7d94e8d13177aaa050113286e93e7b40eae01fbf7c3d9/certifi-2023.7.22-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/certifi@2023.7.22",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "certifi",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "2023.7.22"
},
{
"SPDXID": "SPDXRef-PACKAGE-chardet",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/74/8f/8fc49109009e8d2169d94d72e6b1f4cd45c13d147ba7d6170fb41f22b08f/chardet-5.1.0-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/chardet@5.1.0",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "chardet",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "5.1.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-idna",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/fc/34/3030de6f1370931b9dbb4dad48f6ab1015ab1d32447850b9fc94e60097be/idna-3.4-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/idna@3.4",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "idna",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "3.4"
},
{
"SPDXID": "SPDXRef-PACKAGE-rich",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/fc/1e/482e5eec0b89b593e81d78f819a9412849814e22225842b598908e7ac560/rich-13.4.2-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/rich@13.4.2",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "rich",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "13.4.2"
},
{
"SPDXID": "SPDXRef-PACKAGE-pygments",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/34/a7/37c8d68532ba71549db4212cb036dbd6161b40e463aba336770e80c72f84/Pygments-2.15.1-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/pygments@2.15.1",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "pygments",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "2.15.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-typing-extensions",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/ec/6b/63cc3df74987c36fe26157ee12e09e8f9db4de771e0f3404263117e75b95/typing_extensions-4.7.1-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/typing_extensions@4.7.1",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "typing_extensions",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "4.7.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-resolvelib",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "d2da45d1a8dfee81bdd591647783e340ef3bcb104b54c383f70d422ef5cc7dbf"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/d2/fc/e9ccf0521607bcd244aa0b3fbd574f71b65e9ce6a112c83af988bbbe2e23/resolvelib-1.0.1-py2.py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/resolvelib@1.0.1",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "resolvelib",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "1.0.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-setuptools",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/c7/42/be1c7bbdd83e1bfb160c94b9cafd8e25efc7400346cf7ccdbdb452c467fa/setuptools-68.0.0-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/setuptools@68.0.0",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "setuptools",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "68.0.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-six",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/six@1.16.0",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "six",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "1.16.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-tenacity",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/e7/b0/c23bd61e1b32c9b96fbca996c87784e196a812da8d621d8d04851f6c8181/tenacity-8.2.2-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/tenacity@8.2.2",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "tenacity",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "8.2.2"
},
{
"SPDXID": "SPDXRef-PACKAGE-tomli",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/tomli@2.0.1",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "tomli",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "2.0.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-truststore",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "e37a5642ae9fc48caa8f120b6283d77225d600d224965a672c9e8ef49ce4bb4c"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/20/56/7811d5439b6a56374f274a8672d8f18b4deadadeb3a9f0c86424b98b6f96/truststore-0.8.0-py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/truststore@0.8.0",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "truststore",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "0.8.0"
},
{
"SPDXID": "SPDXRef-PACKAGE-webencodings",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/webencodings@0.5.1",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "webencodings",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "0.5.1"
},
{
"SPDXID": "SPDXRef-PACKAGE-urllib3",
"checksums": [
{
"algorithm": "SHA256",
"checksumValue": "94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"
}
],
"downloadLocation": "https://files.pythonhosted.org/packages/48/fe/a5c6cc46e9fe9171d7ecf0f33ee7aae14642f8d74baa7af4d7840f9358be/urllib3-1.26.17-py2.py3-none-any.whl",
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "pkg:pypi/urllib3@1.26.17",
"referenceType": "purl"
}
],
"licenseConcluded": "MIT",
"name": "urllib3",
"primaryPackagePurpose": "SOURCE",
"versionInfo": "1.26.17"
},
{
"SPDXID": "SPDXRef-PACKAGE-pip",
"checksums": [
@ -1724,6 +2228,126 @@
}
],
"relationships": [
{
"relatedSpdxElement": "SPDXRef-PACKAGE-cachecontrol",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-certifi",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-chardet",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-colorama",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-distlib",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-distro",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-idna",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-msgpack",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-packaging",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-platformdirs",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-pygments",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-pyparsing",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-pyproject-hooks",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-requests",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-resolvelib",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-rich",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-setuptools",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-six",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-tenacity",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-tomli",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-truststore",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-typing-extensions",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-urllib3",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-PACKAGE-webencodings",
"relationshipType": "DEPENDS_ON",
"spdxElementId": "SPDXRef-PACKAGE-pip"
},
{
"relatedSpdxElement": "SPDXRef-FILE-Modules-expat-COPYING",
"relationshipType": "CONTAINS",

273
Tools/build/generate_sbom.py
View File

@ -8,6 +8,7 @@ import pathlib
import subprocess
import sys
import typing
import zipfile
from urllib.request import urlopen
CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent
@ -16,10 +17,16 @@ CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent
# the license expression is a valid SPDX license expression:
# See: https://spdx.org/licenses
ALLOWED_LICENSE_EXPRESSIONS = {
"MIT",
"CC0-1.0",
"Apache-2.0",
"Apache-2.0 OR BSD-2-Clause",
"BSD-2-Clause",
"BSD-3-Clause",
"CC0-1.0",
"ISC",
"LGPL-2.1-only",
"MIT",
"MPL-2.0",
"Python-2.0.1",
}
# Properties which are required for our purposes.
@ -31,14 +38,13 @@ REQUIRED_PROPERTIES_PACKAGE = frozenset([
"checksums",
"licenseConcluded",
"externalRefs",
"originator",
"primaryPackagePurpose",
])
class PackageFiles(typing.NamedTuple):
"""Structure for describing the files of a package"""
include: list[str]
include: list[str] | None
exclude: list[str] | None = None
@ -118,62 +124,209 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]:
return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")])
def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None:
"""pip is a part of a packaging ecosystem (Python, surprise!) so it's actually
automatable to discover the metadata we need like the version and checksums
so let's do that on behalf of our friends at the PyPA.
def fetch_package_metadata_from_pypi(project: str, version: str, filename: str | None = None) -> tuple[str, str] | None:
"""
Fetches the SHA256 checksum and download location from PyPI.
If we're given a filename then we match with that, otherwise we use wheels.
"""
global PACKAGE_TO_FILES
ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled"
pip_wheels = []
# Find the hopefully one pip wheel in the bundled directory.
for wheel_filename in os.listdir(ensurepip_bundled_dir):
if wheel_filename.startswith("pip-"):
pip_wheels.append(wheel_filename)
if len(pip_wheels) != 1:
print("Zero or multiple pip wheels detected in 'Lib/ensurepip/_bundled'")
sys.exit(1)
pip_wheel_filename = pip_wheels[0]
# Add the wheel filename to the list of files so the SBOM file
# and relationship generator can work its magic on the wheel too.
PACKAGE_TO_FILES["pip"] = PackageFiles(
include=[f"Lib/ensurepip/_bundled/{pip_wheel_filename}"]
)
# Wheel filename format puts the version right after the project name.
pip_version = pip_wheel_filename.split("-")[1]
pip_checksum_sha256 = hashlib.sha256(
(ensurepip_bundled_dir / pip_wheel_filename).read_bytes()
).hexdigest()
# Get pip's download location from PyPI. Check that the checksum is correct too.
try:
raw_text = urlopen(f"https://pypi.org/pypi/pip/{pip_version}/json").read()
pip_release_metadata = json.loads(raw_text)
raw_text = urlopen(f"https://pypi.org/pypi/{project}/{version}/json").read()
release_metadata = json.loads(raw_text)
url: dict[str, typing.Any]
# Look for a matching artifact filename and then check
# its remote checksum to the local one.
for url in pip_release_metadata["urls"]:
if url["filename"] == pip_wheel_filename:
for url in release_metadata["urls"]:
# pip can only use Python-only dependencies, so there's
# no risk of picking the 'incorrect' wheel here.
if (
(filename is None and url["packagetype"] == "bdist_wheel")
or (filename is not None and url["filename"] == filename)
):
break
else:
raise ValueError(f"No matching filename on PyPI for '{pip_wheel_filename}'")
if url["digests"]["sha256"] != pip_checksum_sha256:
raise ValueError(f"Local pip checksum doesn't match artifact on PyPI")
raise ValueError(f"No matching filename on PyPI for '{filename}'")
# Successfully found the download URL for the matching artifact.
pip_download_url = url["url"]
download_url = url["url"]
checksum_sha256 = url["digests"]["sha256"]
return download_url, checksum_sha256
except (OSError, ValueError) as e:
print(f"Couldn't fetch pip's metadata from PyPI: {e}")
# Fail if we're running in CI where we should have an internet connection.
error_if(
"CI" in os.environ,
f"Couldn't fetch metadata for project '{project}' from PyPI: {e}"
)
return None
def find_ensurepip_pip_wheel() -> pathlib.Path | None:
"""Try to find the pip wheel bundled in ensurepip. If missing return None"""
ensurepip_bundled_dir = CPYTHON_ROOT_DIR / "Lib/ensurepip/_bundled"
pip_wheels = []
try:
for wheel_filename in os.listdir(ensurepip_bundled_dir):
if wheel_filename.startswith("pip-"):
pip_wheels.append(wheel_filename)
else:
print(f"Unexpected wheel in ensurepip: '{wheel_filename}'")
sys.exit(1)
# Ignore this error, likely caused by downstream distributors
# deleting the 'ensurepip/_bundled' directory.
except FileNotFoundError:
pass
if len(pip_wheels) == 0:
return None
elif len(pip_wheels) > 1:
print("Multiple pip wheels detected in 'Lib/ensurepip/_bundled'")
sys.exit(1)
# Otherwise return the one pip wheel.
return ensurepip_bundled_dir / pip_wheels[0]
def maybe_remove_pip_and_deps_from_sbom(sbom_data: dict[str, typing.Any]) -> None:
"""
Removes pip and its dependencies from the SBOM data
if the pip wheel is removed from ensurepip. This is done
by redistributors of Python and pip.
"""
# If there's a wheel we don't remove anything.
if find_ensurepip_pip_wheel() is not None:
return
# Otherwise we traverse the relationships
# to find dependent packages to remove.
sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip")
sbom_spdx_ids_to_remove = {sbom_pip_spdx_id}
# Find all package SPDXIDs that pip depends on.
for sbom_relationship in sbom_data["relationships"]:
if (
sbom_relationship["relationshipType"] == "DEPENDS_ON"
and sbom_relationship["spdxElementId"] == sbom_pip_spdx_id
):
sbom_spdx_ids_to_remove.add(sbom_relationship["relatedSpdxElement"])
# Remove all the packages and relationships.
sbom_data["packages"] = [
sbom_package for sbom_package in sbom_data["packages"]
if sbom_package["SPDXID"] not in sbom_spdx_ids_to_remove
]
sbom_data["relationships"] = [
sbom_relationship for sbom_relationship in sbom_data["relationships"]
if sbom_relationship["relatedSpdxElement"] not in sbom_spdx_ids_to_remove
]
def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None:
"""pip is a part of a packaging ecosystem (Python, surprise!) so it's actually
automatable to discover the metadata we need like the version and checksums
so let's do that on behalf of our friends at the PyPA. This function also
discovers vendored packages within pip and fetches their metadata.
"""
global PACKAGE_TO_FILES
pip_wheel_filepath = find_ensurepip_pip_wheel()
if pip_wheel_filepath is None:
return # There's no pip wheel, nothing to discover.
# Add the wheel filename to the list of files so the SBOM file
# and relationship generator can work its magic on the wheel too.
PACKAGE_TO_FILES["pip"] = PackageFiles(
include=[str(pip_wheel_filepath.relative_to(CPYTHON_ROOT_DIR))]
)
# Wheel filename format puts the version right after the project name.
pip_version = pip_wheel_filepath.name.split("-")[1]
pip_checksum_sha256 = hashlib.sha256(
pip_wheel_filepath.read_bytes()
).hexdigest()
pip_metadata = fetch_package_metadata_from_pypi(
project="pip",
version=pip_version,
filename=pip_wheel_filepath.name,
)
# We couldn't fetch any metadata from PyPI,
# so we give up on verifying if we're not in CI.
if pip_metadata is None:
return
pip_download_url, pip_actual_sha256 = pip_metadata
if pip_actual_sha256 != pip_checksum_sha256:
raise ValueError("Unexpected")
# Parse 'pip/_vendor/vendor.txt' from the wheel for sub-dependencies.
with zipfile.ZipFile(pip_wheel_filepath) as whl:
vendor_txt_data = whl.read("pip/_vendor/vendor.txt").decode()
# With this version regex we're assuming that pip isn't using pre-releases.
# If any version doesn't match we get a failure below, so we're safe doing this.
version_pin_re = re.compile(r"^([a-zA-Z0-9_.-]+)==([0-9.]*[0-9])$")
sbom_pip_dependency_spdx_ids = set()
for line in vendor_txt_data.splitlines():
line = line.partition("#")[0].strip() # Strip comments and whitespace.
if not line: # Skip empty lines.
continue
# Non-empty lines we must be able to match.
match = version_pin_re.match(line)
error_if(match is None, f"Couldn't parse line from pip vendor.txt: '{line}'")
assert match is not None # Make mypy happy.
# Parse out and normalize the project name.
project_name, project_version = match.groups()
project_name = project_name.lower()
# At this point if pip's metadata fetch succeeded we should
# expect this request to also succeed.
project_metadata = (
fetch_package_metadata_from_pypi(project_name, project_version)
)
assert project_metadata is not None
project_download_url, project_checksum_sha256 = project_metadata
# Update our SBOM data with what we received from PyPI.
# Don't overwrite any existing values.
sbom_project_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{project_name}")
sbom_pip_dependency_spdx_ids.add(sbom_project_spdx_id)
for package in sbom_data["packages"]:
if package["SPDXID"] != sbom_project_spdx_id:
continue
# Only thing missing from this blob is the `licenseConcluded`,
# that needs to be triaged by human maintainers if the list changes.
package.update({
"SPDXID": sbom_project_spdx_id,
"name": project_name,
"versionInfo": project_version,
"downloadLocation": project_download_url,
"checksums": [
{"algorithm": "SHA256", "checksumValue": project_checksum_sha256}
],
"externalRefs": [
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": f"pkg:pypi/{project_name}@{project_version}",
"referenceType": "purl",
},
],
"primaryPackagePurpose": "SOURCE"
})
break
PACKAGE_TO_FILES[project_name] = PackageFiles(include=None)
# Remove pip from the existing SBOM packages if it's there
# and then overwrite its entry with our own generated one.
sbom_pip_spdx_id = spdx_id("SPDXRef-PACKAGE-pip")
sbom_data["packages"] = [
sbom_package
for sbom_package in sbom_data["packages"]
@ -181,7 +334,7 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None:
]
sbom_data["packages"].append(
{
"SPDXID": spdx_id("SPDXRef-PACKAGE-pip"),
"SPDXID": sbom_pip_spdx_id,
"name": "pip",
"versionInfo": pip_version,
"originator": "Organization: Python Packaging Authority",
@ -205,12 +358,27 @@ def discover_pip_sbom_package(sbom_data: dict[str, typing.Any]) -> None:
"primaryPackagePurpose": "SOURCE",
}
)
for sbom_dep_spdx_id in sorted(sbom_pip_dependency_spdx_ids):
sbom_data["relationships"].append({
"spdxElementId": sbom_pip_spdx_id,
"relatedSpdxElement": sbom_dep_spdx_id,
"relationshipType": "DEPENDS_ON"
})
def main() -> None:
sbom_path = CPYTHON_ROOT_DIR / "Misc/sbom.spdx.json"
sbom_data = json.loads(sbom_path.read_bytes())
# Check if pip should be removed if the wheel is missing.
# We can't reset the SBOM relationship data until checking this.
maybe_remove_pip_and_deps_from_sbom(sbom_data)
# We regenerate all of this information. Package information
# should be preserved though since that is edited by humans.
sbom_data["files"] = []
sbom_data["relationships"] = []
# Insert pip's SBOM metadata from the wheel.
discover_pip_sbom_package(sbom_data)
@ -227,9 +395,10 @@ def main() -> None:
"name" not in package,
"Package is missing the 'name' field"
)
missing_required_keys = REQUIRED_PROPERTIES_PACKAGE - set(package.keys())
error_if(
set(package.keys()) != REQUIRED_PROPERTIES_PACKAGE,
f"Package '{package['name']}' is missing required fields",
bool(missing_required_keys),
f"Package '{package['name']}' is missing required fields: {missing_required_keys}",
)
error_if(
package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"),
@ -257,15 +426,11 @@ def main() -> None:
f"License identifier '{license_concluded}' not in SBOM tool allowlist"
)
# Regenerate file information from current data.
sbom_files = []
sbom_relationships = []
# We call 'sorted()' here a lot to avoid filesystem scan order issues.
for name, files in sorted(PACKAGE_TO_FILES.items()):
package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}")
exclude = files.exclude or ()
for include in sorted(files.include):
for include in sorted(files.include or ()):
# Find all the paths and then filter them through .gitignore.
paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True)
paths = filter_gitignored_paths(paths)
@ -285,7 +450,7 @@ def main() -> None:
checksum_sha256 = hashlib.sha256(data).hexdigest()
file_spdx_id = spdx_id(f"SPDXRef-FILE-{path}")
sbom_files.append({
sbom_data["files"].append({
"SPDXID": file_spdx_id,
"fileName": path,
"checksums": [
@ -295,15 +460,13 @@ def main() -> None:
})
# Tie each file back to its respective package.
sbom_relationships.append({
sbom_data["relationships"].append({
"spdxElementId": package_spdx_id,
"relatedSpdxElement": file_spdx_id,
"relationshipType": "CONTAINS",
})
# Update the SBOM on disk
sbom_data["files"] = sbom_files
sbom_data["relationships"] = sbom_relationships
sbom_path.write_text(json.dumps(sbom_data, indent=2, sort_keys=True))