diff options
| -rw-r--r-- | .gitignore | 6 | ||||
| -rw-r--r-- | generate_release.py | 196 | ||||
| -rw-r--r-- | run.bat | 2 |
3 files changed, 202 insertions, 2 deletions
@@ -3,7 +3,11 @@ __pycache__/ debug/ data/ python/ +release/ +youtube-local/ banned_addresses.txt settings.txt - +get-pip.py +latest-dist.zip +*.zip *venv* diff --git a/generate_release.py b/generate_release.py new file mode 100644 index 0000000..962fb00 --- /dev/null +++ b/generate_release.py @@ -0,0 +1,196 @@ +# Generate a windows release and a generated embedded distribution of python +# Latest python version is the argument of the script +# Only works on windows at the moment + +import sys +import urllib +import urllib.request +import subprocess +import shutil +import os + +latest_version = sys.argv[1] +if sys.argv[2] == '-nd': + downloads_enabled = False +elif sys.argv[2] == '-d': + downloads_enabled = True +else: + raise Exception('No download switch specified') + +def check(code): + if code != 0: + raise Exception('Got nonzero exit code from command') + +def log(line): + print('[generate_release.py] ' + line) + +# https://stackoverflow.com/questions/7833715/python-deleting-certain-file-extensions +def remove_files_with_extensions(path, extensions): + for root, dirs, files in os.walk(path): + for file in files: + if os.path.splitext(file)[1] in extensions: + os.remove(os.path.join(root, file)) + +# ---------- Get current release version, for later ---------- +log('Getting current release version') +describe_result = subprocess.run(['git', 'describe', '--tags'], stdout=subprocess.PIPE) +if describe_result.returncode != 0: + raise Exception('Git describe failed') + +release_tag = describe_result.stdout.strip().decode('ascii') + + +# ----------- Make copy of youtube-local files using git ----------- + +if os.path.exists('./youtube-local'): + log('Removing old release') + shutil.rmtree('./youtube-local') + +# Export git repository - this will ensure .git and things in gitignore won't +# be included. Git only supports exporting archive formats, not into +# directories, so pipe into 7z to put it into .\youtube-local (not to be +# confused with working directory. I'm calling it the same thing so it will +# have that name when extracted from the final release zip archive) +log('Making copy of youtube-local files') +check(os.system('git archive --format tar master | 7z x -si -ttar -oyoutube-local')) + +if len(os.listdir('./youtube-local')) == 0: + raise Exception('Failed to copy youtube-local files') + + +# ----------- Generate embedded python distribution ----------- +os.environ['PYTHONDONTWRITEBYTECODE'] = '1' # *.pyc files double the size of the distribution +get_pip_url = 'https://bootstrap.pypa.io/get-pip.py' +latest_dist_url = 'https://www.python.org/ftp/python/' + latest_version + '/python-' + latest_version + '-embed-win32.zip' + +if downloads_enabled: + log('Downloading get-pip.py...') + get_pip = urllib.request.urlopen(get_pip_url).read() + log('Finished downloading get-pip.py') + + with open('./get-pip.py', 'wb') as f: + f.write(get_pip) + + log('Downloading latest python distribution...') + latest_dist= urllib.request.urlopen(latest_dist_url).read() + log('Finished downloading python distribution') + + with open('./latest-dist.zip', 'wb') as f: + f.write(latest_dist) + +if os.path.exists('./python'): + log('Removing old python distribution') + shutil.rmtree('./python') + + +log('Extracting python distribution') + +check(os.system(r'7z -y x -opython latest-dist.zip')) + +log('Executing get-pip.py') +os.system(r'.\python\python.exe -I get-pip.py') + +''' +# Explanation of .pth, ._pth, and isolated mode + +## Isolated mode + We want to run in what is called isolated mode, given by the switch -I. +This mode prevents the embedded python distribution from searching in +global directories for imports + + For example, if a user has `C:\Python37` and the embedded distribution is +the same version, importing something using the embedded distribution will +search `C:\Python37\Libs\site-packages`. This is not desirable because it +means I might forget to distribute a dependency if I have it installed +globally and I don't see any import errors. It also means that an outdated +package might override the one being distributed and cause other problems. + + Isolated mode also means global environment variables and registry +entries will be ignored + +## The trouble with isolated mode + Isolated mode also prevents the current working directory (cwd) from +being added to `sys.path`. `sys.path` is the list of directories python will +search in for imports. In non-isolated mode this is automatically populated +with the cwd, `site-packages`, the directory of the python executable, etc. + +# How to get the cwd into sys.path in isolated mode + The hack to get this to work is to use a .pth file. Normally, these files +are just an additional list of directories to be added to `sys.path`. +However, they also allow arbitrary code execution on lines beginning with +`import ` (see https://docs.python.org/3/library/site.html). So, we simply +add `import sys; sys.path.insert(0, '')` to add the cwd to path. `''` is +shorthand for the cwd. See https://bugs.python.org/issue33698#msg318272 + +# ._pth files in the embedded distribution +A python37._pth file is included in the embedded distribution. The presence +of tis file causes the embedded distribution to always use isolated mode +(which we want). They are like .pth files, except they do not allow the +arbitrary code execution trick. In my experimentation, I found that they +prevent .pth files from loading. So the ._pth file will have to be removed +and replaced with a .pth. Isolated mode will have to be specified manually. +''' + +log('Removing ._pth') +major_release = latest_version.split('.')[1] +os.remove(r'.\python\python3' + major_release + '._pth') + +log('Adding path_fixes.pth') +with open(r'.\python\path_fixes.pth', 'w', encoding='utf-8') as f: + f.write("import sys; sys.path.insert(0, '')\n") + + +'''# python3x._pth file tells the python executable where to look for files +# Need to add the directory where packages are installed, +# and the parent directory (which is where the youtube-local files are) +major_release = latest_version.split('.')[1] +with open('./python/python3' + major_release + '._pth', 'a', encoding='utf-8') as f: + f.write('.\\Lib\\site-packages\n') + f.write('..\n')''' + +log('Inserting Microsoft C Runtime') +check(os.system(r'copy C:\Windows\SysWOW64\msvcp140.dll .\python\msvcp140.dll')) + +log('Installing dependencies') +check(os.system(r'.\python\python.exe -I -m pip install --no-compile -r .\requirements.txt')) + +log('Uninstalling unnecessary gevent stuff') +check(os.system(r'.\python\python.exe -I -m pip uninstall --yes cffi pycparser')) +shutil.rmtree(r'./python/Lib/site-packages/gevent/tests') +shutil.rmtree(r'./python/Lib/site-packages/gevent/testing') +remove_files_with_extensions(r'./python/Lib/site-packages/gevent', ['.html']) # bloated html documentation + +log('Uninstalling pip and others') +check(os.system(r'.\python\python.exe -I -m pip uninstall --yes pip setuptools wheel')) + +log('Removing pyc files') # Have to do this because get-pip and some packages don't respect --no-compile +remove_files_with_extensions(r'./python', ['.pyc']) + +log('Removing dist-info and __pycache__') +for root, dirs, files in os.walk(r'./python'): + for dir in dirs: + if dir == '__pycache__' or dir.endswith('.dist-info'): + shutil.rmtree(os.path.join(root, dir)) + + +'''log('Removing get-pip.py and zipped distribution') +os.remove(r'.\get-pip.py') +os.remove(r'.\latest-dist.zip')''' + +print() +log('Finished generating python distribution') + +# ----------- Copy generated distribution into release folder ----------- +log('Copying python distribution into release folder') +shutil.copytree(r'.\python', r'.\youtube-local\python') + +# ----------- Create release zip ----------- +output_filename = 'youtube-local-' + release_tag + '-windows.zip' +if os.path.exists('./' + output_filename): + log('Removing previous zipped release') + os.remove('.\\' + output_filename) +log('Zipping release') +check(os.system(r'7z -mx=9 a ' + output_filename + ' .\youtube-local')) + +print('\n') +log('Finished') @@ -1,4 +1,4 @@ @echo off -.\python\python.exe .\server.py +.\python\python.exe -I .\server.py echo Press any key to quit... PAUSE > nul |