aboutsummaryrefslogtreecommitdiffstats
path: root/generate_release.py
diff options
context:
space:
mode:
Diffstat (limited to 'generate_release.py')
-rw-r--r--generate_release.py173
1 files changed, 127 insertions, 46 deletions
diff --git a/generate_release.py b/generate_release.py
index 962fb00..46e91bb 100644
--- a/generate_release.py
+++ b/generate_release.py
@@ -1,6 +1,8 @@
# Generate a windows release and a generated embedded distribution of python
-# Latest python version is the argument of the script
-# Only works on windows at the moment
+# Latest python version is the argument of the script (or oldwin for
+# vista, 7 and 32-bit versions)
+# Requirements: 7z, git
+# wine is required in order to build on Linux
import sys
import urllib
@@ -8,21 +10,30 @@ import urllib.request
import subprocess
import shutil
import os
+import hashlib
latest_version = sys.argv[1]
-if sys.argv[2] == '-nd':
- downloads_enabled = False
-elif sys.argv[2] == '-d':
- downloads_enabled = True
+if len(sys.argv) > 2:
+ bitness = sys.argv[2]
else:
- raise Exception('No download switch specified')
+ bitness = '64'
+
+if latest_version == 'oldwin':
+ bitness = '32'
+ latest_version = '3.7.9'
+ suffix = 'windows-vista-7-only'
+else:
+ suffix = 'windows'
def check(code):
if code != 0:
raise Exception('Got nonzero exit code from command')
+def check_subp(x):
+ if x.returncode != 0:
+ raise Exception('Got nonzero exit code from command')
def log(line):
- print('[generate_release.py] ' + line)
+ print(f'[generate_release.py] {line}')
# https://stackoverflow.com/questions/7833715/python-deleting-certain-file-extensions
def remove_files_with_extensions(path, extensions):
@@ -31,6 +42,41 @@ def remove_files_with_extensions(path, extensions):
if os.path.splitext(file)[1] in extensions:
os.remove(os.path.join(root, file))
+def download_if_not_exists(file_name, url, sha256=None):
+ if not os.path.exists(f'./{file_name}'):
+ # Reject non-https URLs so a mistaken constant cannot cause a
+ # plaintext download (bandit B310 hardening).
+ if not url.startswith('https://'):
+ raise Exception(f'Refusing to download over non-https URL: {url}')
+ log(f'Downloading {file_name}..')
+ data = urllib.request.urlopen(url).read()
+ log(f'Finished downloading {file_name}')
+ with open(f'./{file_name}', 'wb') as f:
+ f.write(data)
+ if sha256:
+ digest = hashlib.sha256(data).hexdigest()
+ if digest != sha256:
+ log(f'Error: {file_name} has wrong hash: {digest}')
+ sys.exit(1)
+ else:
+ log(f'Using existing {file_name}')
+
+def wine_run_shell(command):
+ # Keep argv-style invocation (no shell) to avoid command injection.
+ if os.name == 'posix':
+ parts = ['wine'] + command.replace('\\', '/').split()
+ elif os.name == 'nt':
+ parts = command.split()
+ else:
+ raise Exception('Unsupported OS')
+ check(subprocess.run(parts).returncode)
+
+def wine_run(command_parts):
+ if os.name == 'posix':
+ command_parts = ['wine',] + command_parts
+ if subprocess.run(command_parts).returncode != 0:
+ raise Exception('Got nonzero exit code from command')
+
# ---------- Get current release version, for later ----------
log('Getting current release version')
describe_result = subprocess.run(['git', 'describe', '--tags'], stdout=subprocess.PIPE)
@@ -40,43 +86,67 @@ if describe_result.returncode != 0:
release_tag = describe_result.stdout.strip().decode('ascii')
-# ----------- Make copy of youtube-local files using git -----------
+# ----------- Make copy of yt-local files using git -----------
-if os.path.exists('./youtube-local'):
+if os.path.exists('./yt-local'):
log('Removing old release')
- shutil.rmtree('./youtube-local')
+ shutil.rmtree('./yt-local')
# Export git repository - this will ensure .git and things in gitignore won't
# be included. Git only supports exporting archive formats, not into
-# directories, so pipe into 7z to put it into .\youtube-local (not to be
+# directories, so pipe into 7z to put it into .\yt-local (not to be
# confused with working directory. I'm calling it the same thing so it will
# have that name when extracted from the final release zip archive)
-log('Making copy of youtube-local files')
-check(os.system('git archive --format tar master | 7z x -si -ttar -oyoutube-local'))
-
-if len(os.listdir('./youtube-local')) == 0:
- raise Exception('Failed to copy youtube-local files')
+log('Making copy of yt-local files')
+# Avoid the shell: pipe `git archive` into 7z directly via subprocess.
+_git_archive = subprocess.Popen(
+ ['git', 'archive', '--format', 'tar', 'master'],
+ stdout=subprocess.PIPE,
+)
+_sevenz = subprocess.Popen(
+ ['7z', 'x', '-si', '-ttar', '-oyt-local'],
+ stdin=_git_archive.stdout,
+)
+_git_archive.stdout.close()
+_sevenz.wait()
+_git_archive.wait()
+check(_sevenz.returncode)
+check(_git_archive.returncode)
+
+if len(os.listdir('./yt-local')) == 0:
+ raise Exception('Failed to copy yt-local files')
# ----------- Generate embedded python distribution -----------
os.environ['PYTHONDONTWRITEBYTECODE'] = '1' # *.pyc files double the size of the distribution
get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
-latest_dist_url = 'https://www.python.org/ftp/python/' + latest_version + '/python-' + latest_version + '-embed-win32.zip'
-
-if downloads_enabled:
- log('Downloading get-pip.py...')
- get_pip = urllib.request.urlopen(get_pip_url).read()
- log('Finished downloading get-pip.py')
+latest_dist_url = f'https://www.python.org/ftp/python/{latest_version}/python-{latest_version}'
+if bitness == '32':
+ latest_dist_url += '-embed-win32.zip'
+else:
+ latest_dist_url += '-embed-amd64.zip'
+
+# I've verified that all the dlls in the following are signed by Microsoft.
+# Using this because Microsoft only provides installers whose files can't be
+# extracted without a special tool.
+if bitness == '32':
+ visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x86.7z'
+ visual_c_runtime_sha256 = '2549eb4d2ce4cf3a87425ea01940f74368bf1cda378ef8a8a1f1a12ed59f1547'
+ visual_c_name = 'vc15_(14.10.25017.0)_2017_x86.7z'
+ visual_c_path_to_dlls = 'runtime_minimum/System'
+else:
+ visual_c_runtime_url = 'https://github.com/yuempek/vc-archive/raw/master/archives/vc15_(14.10.25017.0)_2017_x64.7z'
+ visual_c_runtime_sha256 = '4f00b824c37e1017a93fccbd5775e6ee54f824b6786f5730d257a87a3d9ce921'
+ visual_c_name = 'vc15_(14.10.25017.0)_2017_x64.7z'
+ visual_c_path_to_dlls = 'runtime_minimum/System64'
- with open('./get-pip.py', 'wb') as f:
- f.write(get_pip)
+download_if_not_exists('get-pip.py', get_pip_url)
- log('Downloading latest python distribution...')
- latest_dist= urllib.request.urlopen(latest_dist_url).read()
- log('Finished downloading python distribution')
+python_dist_name = f'python-dist-{latest_version}-{bitness}.zip'
- with open('./latest-dist.zip', 'wb') as f:
- f.write(latest_dist)
+download_if_not_exists(python_dist_name, latest_dist_url)
+download_if_not_exists(visual_c_name,
+ visual_c_runtime_url, sha256=visual_c_runtime_sha256)
if os.path.exists('./python'):
log('Removing old python distribution')
@@ -85,17 +155,17 @@ if os.path.exists('./python'):
log('Extracting python distribution')
-check(os.system(r'7z -y x -opython latest-dist.zip'))
+check_subp(subprocess.run(['7z', '-y', 'x', '-opython', python_dist_name]))
log('Executing get-pip.py')
-os.system(r'.\python\python.exe -I get-pip.py')
+wine_run(['./python/python.exe', '-I', 'get-pip.py'])
'''
# Explanation of .pth, ._pth, and isolated mode
## Isolated mode
We want to run in what is called isolated mode, given by the switch -I.
-This mode prevents the embedded python distribution from searching in
+This mode prevents the embedded python distribution from searching in
global directories for imports
For example, if a user has `C:\Python37` and the embedded distribution is
@@ -133,35 +203,46 @@ and replaced with a .pth. Isolated mode will have to be specified manually.
log('Removing ._pth')
major_release = latest_version.split('.')[1]
-os.remove(r'.\python\python3' + major_release + '._pth')
+os.remove(rf'./python/python3{major_release}._pth')
log('Adding path_fixes.pth')
-with open(r'.\python\path_fixes.pth', 'w', encoding='utf-8') as f:
+with open(r'./python/path_fixes.pth', 'w', encoding='utf-8') as f:
f.write("import sys; sys.path.insert(0, '')\n")
'''# python3x._pth file tells the python executable where to look for files
# Need to add the directory where packages are installed,
-# and the parent directory (which is where the youtube-local files are)
+# and the parent directory (which is where the yt-local files are)
major_release = latest_version.split('.')[1]
-with open('./python/python3' + major_release + '._pth', 'a', encoding='utf-8') as f:
+with open(rf'./python/python3{major_release}._pth', 'a', encoding='utf-8') as f:
f.write('.\\Lib\\site-packages\n')
f.write('..\n')'''
log('Inserting Microsoft C Runtime')
-check(os.system(r'copy C:\Windows\SysWOW64\msvcp140.dll .\python\msvcp140.dll'))
+check_subp(subprocess.run([r'7z', '-y', 'e', '-opython', visual_c_name, visual_c_path_to_dlls]))
log('Installing dependencies')
-check(os.system(r'.\python\python.exe -I -m pip install --no-compile -r .\requirements.txt'))
+pip_install = ['./python/python.exe', '-I', '-m', 'pip', 'install', '--no-compile']
+if os.name == 'posix':
+ # Wine's isolated build environment can't import setuptools.build_meta,
+ # so we install build tools first and disable isolation.
+ wine_run(pip_install + ['setuptools', 'wheel'])
+ wine_run(pip_install + ['--no-build-isolation', '-r', './requirements.txt'])
+else:
+ wine_run(pip_install + ['-r', './requirements.txt'])
log('Uninstalling unnecessary gevent stuff')
-check(os.system(r'.\python\python.exe -I -m pip uninstall --yes cffi pycparser'))
+wine_run(['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes', 'cffi', 'pycparser'])
shutil.rmtree(r'./python/Lib/site-packages/gevent/tests')
shutil.rmtree(r'./python/Lib/site-packages/gevent/testing')
remove_files_with_extensions(r'./python/Lib/site-packages/gevent', ['.html']) # bloated html documentation
log('Uninstalling pip and others')
-check(os.system(r'.\python\python.exe -I -m pip uninstall --yes pip setuptools wheel'))
+pip_uninstall = ['./python/python.exe', '-I', '-m', 'pip', 'uninstall', '--yes']
+if os.name == 'posix':
+ wine_run(pip_uninstall + ['pip', 'setuptools', 'wheel'])
+else:
+ wine_run(pip_uninstall + ['pip', 'wheel'])
log('Removing pyc files') # Have to do this because get-pip and some packages don't respect --no-compile
remove_files_with_extensions(r'./python', ['.pyc'])
@@ -182,15 +263,15 @@ log('Finished generating python distribution')
# ----------- Copy generated distribution into release folder -----------
log('Copying python distribution into release folder')
-shutil.copytree(r'.\python', r'.\youtube-local\python')
+shutil.copytree(r'./python', r'./yt-local/python')
# ----------- Create release zip -----------
-output_filename = 'youtube-local-' + release_tag + '-windows.zip'
-if os.path.exists('./' + output_filename):
+output_filename = f'yt-local-{release_tag}-{suffix}.zip'
+if os.path.exists(f'./{output_filename}'):
log('Removing previous zipped release')
- os.remove('.\\' + output_filename)
+ os.remove(f'./{output_filename}')
log('Zipping release')
-check(os.system(r'7z -mx=9 a ' + output_filename + ' .\youtube-local'))
+check_subp(subprocess.run(['7z', '-mx=9', 'a', output_filename, './yt-local']))
print('\n')
log('Finished')
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-19 12:49:53 +0000