[build] Improve release process (#880)

* Automate more of the release process by animelover1984, pukkandan - closes #823 * Fix sha256 by nihil-admirari - closes #385 * Bring back brew taps by nao20010128nao #865 * Provide `--onedir` zip for windows by pukkandan - Closes #1024, #661, #705 and #890 Authored by: pukkandan, animelover1984, nihil-admirari, nao20010128nao
3 years ago · 4c88ff87fc
parent e27cc5d864
commit 4c88ff87fc
5 changed files with 162 additions and 82 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -12,11 +12,15 @@ jobs:
    outputs:
      ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }}
      upload_url: ${{ steps.create_release.outputs.upload_url }}
-      sha256_unix: ${{ steps.sha256_file.outputs.sha256_unix }}
-      sha512_unix: ${{ steps.sha512_file.outputs.sha512_unix }}
+      sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }}
+      sha512_bin: ${{ steps.sha512_bin.outputs.sha512_bin }}
+      sha256_tar: ${{ steps.sha256_tar.outputs.sha256_tar }}
+      sha512_tar: ${{ steps.sha512_tar.outputs.sha512_tar }}

    steps:
    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
@ -25,11 +29,76 @@ jobs:
      run: sudo apt-get -y install zip pandoc man
    - name: Bump version
      id: bump_version
-      run: python devscripts/update-version.py
+      run: |
+        python devscripts/update-version.py
+        make issuetemplates
    - name: Print version
      run: echo "${{ steps.bump_version.outputs.ytdlp_version }}"
+    - name: Update master
+      id: push_update
+      run: |
+        git config --global user.email "${{ github.event.pusher.email }}"
+        git config --global user.name "${{ github.event.pusher.name }}"
+        git add -u
+        git commit -m "[version] update" -m ":ci skip all"
+        git pull --rebase origin ${{ github.event.repository.master_branch }}
+        git push origin ${{ github.event.ref }}:${{ github.event.repository.master_branch }}
+        echo ::set-output name=head_sha::$(git rev-parse HEAD)
+    - name: Get Changelog
+      id: get_changelog
+      run: |
+        changelog=$(cat Changelog.md | grep -oPz '(?s)(?<=### ${{ steps.bump_version.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)')
+        echo "changelog<<EOF" >> $GITHUB_ENV
+        echo "$changelog" >> $GITHUB_ENV
+        echo "EOF" >> $GITHUB_ENV
    - name: Run Make
      run: make all tar
+    - name: Get SHA2-256SUMS for yt-dlp
+      id: sha256_bin
+      run: echo "::set-output name=sha256_bin::$(sha256sum yt-dlp | awk '{print $1}')"
+    - name: Get SHA2-256SUMS for yt-dlp.tar.gz
+      id: sha256_tar
+      run: echo "::set-output name=sha256_tar::$(sha256sum yt-dlp.tar.gz | awk '{print $1}')"
+    - name: Get SHA2-512SUMS for yt-dlp
+      id: sha512_bin
+      run: echo "::set-output name=sha512_bin::$(sha512sum yt-dlp | awk '{print $1}')"
+    - name: Get SHA2-512SUMS for yt-dlp.tar.gz
+      id: sha512_tar
+      run: echo "::set-output name=sha512_tar::$(sha512sum yt-dlp.tar.gz | awk '{print $1}')"
+    - name: Install SSH private key
+      env:
+        BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
+      if: "env.BREW_TOKEN != ''"
+      uses: webfactory/ssh-agent@v0.5.3
+      with:
+          ssh-private-key: ${{ env.BREW_TOKEN }}
+    - name: Update Homebrew Formulae
+      env:
+        BREW_TOKEN: ${{ secrets.BREW_TOKEN }}
+      if: "env.BREW_TOKEN != ''"
+      run: |
+        git clone git@github.com:yt-dlp/homebrew-taps taps/
+        python3 devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ steps.bump_version.outputs.ytdlp_version }}"
+        git -C taps/ config user.name github-actions
+        git -C taps/ config user.email github-actions@example.com
+        git -C taps/ commit -am 'yt-dlp: ${{ steps.bump_version.outputs.ytdlp_version }}'
+        git -C taps/ push
+    - name: Install dependencies for pypi
+      env:
+        PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+      if: "env.PYPI_TOKEN != ''"
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine
+    - name: Build and publish on pypi
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+      if: "env.TWINE_PASSWORD != ''"
+      run: |
+        rm -rf dist/*
+        python setup.py sdist bdist_wheel
+        twine upload dist/*
    - name: Create Release
      id: create_release
      uses: actions/create-release@v1
@ -38,9 +107,10 @@ jobs:
      with:
        tag_name: ${{ steps.bump_version.outputs.ytdlp_version }}
        release_name: yt-dlp ${{ steps.bump_version.outputs.ytdlp_version }}
+        commitish: ${{ steps.push_update.outputs.head_sha }}
        body: |
          Changelog:
-          PLACEHOLDER
+          ${{ env.changelog }}
        draft: false
        prerelease: false
    - name: Upload yt-dlp Unix binary
@ -62,36 +132,16 @@ jobs:
        asset_path: ./yt-dlp.tar.gz
        asset_name: yt-dlp.tar.gz
        asset_content_type: application/gzip
-    - name: Get SHA2-256SUMS for yt-dlp
-      id: sha256_file
-      run: echo "::set-output name=sha256_unix::$(sha256sum yt-dlp | awk '{print $1}')"
-    - name: Get SHA2-512SUMS for yt-dlp
-      id: sha512_file
-      run: echo "::set-output name=sha512_unix::$(sha512sum yt-dlp | awk '{print $1}')"
-    - name: Install dependencies for pypi
-      env:
-        PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
-      if: "env.PYPI_TOKEN != ''"
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish on pypi
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-      if: "env.TWINE_PASSWORD != ''"
-      run: |
-        rm -rf dist/*
-        python setup.py sdist bdist_wheel
-        twine upload dist/*

  build_windows:
    runs-on: windows-latest
    needs: build_unix

    outputs:
-      sha256_windows: ${{ steps.sha256_file_win.outputs.sha256_windows }}
-      sha512_windows: ${{ steps.sha512_file_win.outputs.sha512_windows }}
+      sha256_win: ${{ steps.sha256_win.outputs.sha256_win }}
+      sha512_win: ${{ steps.sha512_win.outputs.sha512_win }}
+      sha256_win_zip: ${{ steps.sha256_win_zip.outputs.sha256_win_zip }}
+      sha512_win_zip: ${{ steps.sha512_win_zip.outputs.sha512_win_zip }}

    steps:
    - uses: actions/checkout@v2
@ -104,7 +154,7 @@ jobs:
      run: python -m pip install --upgrade pip setuptools wheel
    - name: Install Requirements
      # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
-      run: pip install "https://yt-dlp.github.io/pyinstaller-builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets
+      run: pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets
    - name: Bump version
      id: bump_version
      run: python devscripts/update-version.py
@ -123,19 +173,41 @@ jobs:
        asset_name: yt-dlp.exe
        asset_content_type: application/vnd.microsoft.portable-executable
    - name: Get SHA2-256SUMS for yt-dlp.exe
-      id: sha256_file_win
-      run: echo "::set-output name=sha256_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())"
+      id: sha256_win
+      run: echo "::set-output name=sha256_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())"
    - name: Get SHA2-512SUMS for yt-dlp.exe
-      id: sha512_file_win
-      run: echo "::set-output name=sha512_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())"
+      id: sha512_win
+      run: echo "::set-output name=sha512_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())"
+    - name: Run PyInstaller Script with --onedir
+      run: python pyinst.py 64 --onedir
+    - uses: papeloto/action-zip@v1
+      with:
+        files: ./dist/yt-dlp
+        dest: ./dist/yt-dlp.zip
+    - name: Upload yt-dlp.zip Windows onedir
+      id: upload-release-windows-zip
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.build_unix.outputs.upload_url }}
+        asset_path: ./dist/yt-dlp.zip
+        asset_name: yt-dlp.zip
+        asset_content_type: application/zip
+    - name: Get SHA2-256SUMS for yt-dlp.zip
+      id: sha256_win_zip
+      run: echo "::set-output name=sha256_win_zip::$((Get-FileHash dist\yt-dlp.zip -Algorithm SHA256).Hash.ToLower())"
+    - name: Get SHA2-512SUMS for yt-dlp.zip
+      id: sha512_win_zip
+      run: echo "::set-output name=sha512_win_zip::$((Get-FileHash dist\yt-dlp.zip -Algorithm SHA512).Hash.ToLower())"

  build_windows32:
    runs-on: windows-latest
    needs: [build_unix, build_windows]

    outputs:
-      sha256_windows32: ${{ steps.sha256_file_win32.outputs.sha256_windows32 }}
-      sha512_windows32: ${{ steps.sha512_file_win32.outputs.sha512_windows32 }}
+      sha256_win32: ${{ steps.sha256_win32.outputs.sha256_win32 }}
+      sha512_win32: ${{ steps.sha512_win32.outputs.sha512_win32 }}

    steps:
    - uses: actions/checkout@v2
@ -148,7 +220,7 @@ jobs:
    - name: Upgrade pip and enable wheel support
      run: python -m pip install --upgrade pip setuptools wheel
    - name: Install Requirements
-      run: pip install "https://yt-dlp.github.io/pyinstaller-builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets
+      run: pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodome websockets
    - name: Bump version
      id: bump_version
      run: python devscripts/update-version.py
@ -167,11 +239,11 @@ jobs:
        asset_name: yt-dlp_x86.exe
        asset_content_type: application/vnd.microsoft.portable-executable
    - name: Get SHA2-256SUMS for yt-dlp_x86.exe
-      id: sha256_file_win32
-      run: echo "::set-output name=sha256_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())"
+      id: sha256_win32
+      run: echo "::set-output name=sha256_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())"
    - name: Get SHA2-512SUMS for yt-dlp_x86.exe
-      id: sha512_file_win32
-      run: echo "::set-output name=sha512_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())"
+      id: sha512_win32
+      run: echo "::set-output name=sha512_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())"

  finish:
    runs-on: ubuntu-latest
@ -180,15 +252,17 @@ jobs:
    steps:
    - name: Make SHA2-256SUMS file
      env:
-        SHA256_WINDOWS: ${{ needs.build_windows.outputs.sha256_windows }}
-        SHA256_WINDOWS32: ${{ needs.build_windows32.outputs.sha256_windows32 }}
-        SHA256_UNIX: ${{ needs.build_unix.outputs.sha256_unix }}
-        YTDLP_VERSION: ${{ needs.build_unix.outputs.ytdlp_version }}
+        SHA256_WIN: ${{ needs.build_windows.outputs.sha256_win }}
+        SHA256_WIN_ZIP: ${{ needs.build_windows.outputs.sha256_win_zip }}
+        SHA256_WIN32: ${{ needs.build_windows32.outputs.sha256_win32 }}
+        SHA256_BIN: ${{ needs.build_unix.outputs.sha256_bin }}
+        SHA256_TAR: ${{ needs.build_unix.outputs.sha256_tar }}
      run: |
-        echo "version:${{ env.YTDLP_VERSION }}" >> SHA2-256SUMS
-        echo "yt-dlp.exe:${{ env.SHA256_WINDOWS }}" >> SHA2-256SUMS
-        echo "yt-dlp_x86.exe:${{ env.SHA256_WINDOWS32 }}" >> SHA2-256SUMS
-        echo "yt-dlp:${{ env.SHA256_UNIX }}" >> SHA2-256SUMS
+        echo "${{ env.SHA256_WIN }}  yt-dlp.exe" >> SHA2-256SUMS
+        echo "${{ env.SHA256_WIN32 }}  yt-dlp_x86.exe" >> SHA2-256SUMS
+        echo "${{ env.SHA256_BIN }}  yt-dlp" >> SHA2-256SUMS
+        echo "${{ env.SHA256_TAR }}  yt-dlp.tar.gz" >> SHA2-256SUMS
+        echo "${{ env.SHA256_WIN_ZIP }}  yt-dlp.zip" >> SHA2-256SUMS
    - name: Upload 256SUMS file
      id: upload-sums
      uses: actions/upload-release-asset@v1
@ -201,13 +275,17 @@ jobs:
        asset_content_type: text/plain
    - name: Make SHA2-512SUMS file
      env:
-        SHA512_WINDOWS: ${{ needs.build_windows.outputs.sha512_windows }}
-        SHA512_WINDOWS32: ${{ needs.build_windows32.outputs.sha512_windows32 }}
-        SHA512_UNIX: ${{ needs.build_unix.outputs.sha512_unix }}
+        SHA512_WIN: ${{ needs.build_windows.outputs.sha512_win }}
+        SHA512_WIN_ZIP: ${{ needs.build_windows.outputs.sha512_win_zip }}
+        SHA512_WIN32: ${{ needs.build_windows32.outputs.sha512_win32 }}
+        SHA512_BIN: ${{ needs.build_unix.outputs.sha512_bin }}
+        SHA512_TAR: ${{ needs.build_unix.outputs.sha512_tar }}
      run: |
-        echo "${{ env.SHA512_WINDOWS }}  yt-dlp.exe" >> SHA2-512SUMS
-        echo "${{ env.SHA512_WINDOWS32 }}  yt-dlp_x86.exe" >> SHA2-512SUMS
-        echo "${{ env.SHA512_UNIX }}  yt-dlp" >> SHA2-512SUMS
+        echo "${{ env.SHA512_WIN }}  yt-dlp.exe" >> SHA2-512SUMS
+        echo "${{ env.SHA512_WIN32 }}  yt-dlp_x86.exe" >> SHA2-512SUMS
+        echo "${{ env.SHA512_BIN }}  yt-dlp" >> SHA2-512SUMS
+        echo "${{ env.SHA512_TAR }}  yt-dlp.tar.gz" >> SHA2-512SUMS
+        echo "${{ env.SHA512_WIN_ZIP }}  yt-dlp.zip" >> SHA2-512SUMS
    - name: Upload 512SUMS file
      id: upload-512sums
      uses: actions/upload-release-asset@v1
--- a/Changelog.md
+++ b/Changelog.md
@ -7,14 +7,9 @@
 * Update Changelog.md and CONTRIBUTORS
 * Change "Merged with ytdl" version in Readme.md if needed
 * Add new/fixed extractors in "new features" section of Readme.md
-* Commit to master as `Release <version>`
+* Commit as `Release <version>`
 * Push to origin/release using `git push origin master:release`
    build task will now run
-* Update version.py using `devscripts\update-version.py`
-* Run `make issuetemplates`
-* Commit to master as `[version] update :ci skip all`
-* Push to origin/master
-* Update changelog in /releases

 -->

--- a/pyinst.py
+++ b/pyinst.py
@ -15,9 +15,11 @@ import PyInstaller.__main__

 arch = sys.argv[1] if len(sys.argv) > 1 else platform.architecture()[0][:2]
 assert arch in ('32', '64')
-print('Building %sbit version' % arch)
 _x86 = '_x86' if arch == '32' else ''

+opts = sys.argv[2:] or ['--onefile']
+print(f'Building {arch}bit version with options {opts}')
+
 FILE_DESCRIPTION = 'yt-dlp%s' % (' (32 Bit)' if _x86 else '')

 # root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
@ -72,11 +74,12 @@ excluded_modules = ['test', 'ytdlp_plugins', 'youtube-dl', 'youtube-dlc']

 PyInstaller.__main__.run([
    '--name=yt-dlp%s' % _x86,
-    '--onefile',
    '--icon=devscripts/logo.ico',
    *[f'--exclude-module={module}' for module in excluded_modules],
    *[f'--hidden-import={module}' for module in dependancies],
    '--upx-exclude=vcruntime140.dll',
+    '--noconfirm',
+    *opts,
    'yt_dlp/__main__.py',
 ])
 SetVersion('dist/yt-dlp%s.exe' % _x86, VERSION_FILE)
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@ -27,7 +27,6 @@ import traceback
 import random

 from string import ascii_letters
-from zipimport import zipimporter

 from .compat import (
    compat_basestring,
@ -143,6 +142,7 @@ from .postprocessor import (
    FFmpegPostProcessor,
    MoveFilesAfterDownloadPP,
 )
+from .update import detect_variant
 from .version import __version__

 if compat_os_name == 'nt':
@ -3266,12 +3266,8 @@ class YoutubeDL(object):
                self.get_encoding()))
        write_string(encoding_str, encoding=None)

-        source = (
-            '(exe)' if hasattr(sys, 'frozen')
-            else '(zip)' if isinstance(globals().get('__loader__'), zipimporter)
-            else '(source)' if os.path.basename(sys.argv[0]) == '__main__.py'
-            else '')
-        self._write_string('[debug] yt-dlp version %s %s\n' % (__version__, source))
+        source = detect_variant()
+        self._write_string('[debug] yt-dlp version %s%s\n' % (__version__, '' if source == 'unknown' else f' ({source})'))
        if _LAZY_LOADER:
            self._write_string('[debug] Lazy loading extractors enabled\n')
        if _PLUGIN_CLASSES:
--- a/yt_dlp/update.py
+++ b/yt_dlp/update.py
@ -31,6 +31,18 @@ def rsa_verify(message, signature, key):
 '''


+def detect_variant():
+    if hasattr(sys, 'frozen') and getattr(sys, '_MEIPASS', None):
+        if sys._MEIPASS == os.path.dirname(sys.executable):
+            return 'dir'
+        return 'exe'
+    elif isinstance(globals().get('__loader__'), zipimporter):
+        return 'zip'
+    elif os.path.basename(sys.argv[0]) == '__main__.py':
+        return 'source'
+    return 'unknown'
+
+
 def update_self(to_screen, verbose, opener):
    ''' Exists for backward compatibility. Use run_update(ydl) instead '''

@ -87,13 +99,14 @@ def run_update(ydl):
                h.update(mv[:n])
        return h.hexdigest()

-    err = None
-    if isinstance(globals().get('__loader__'), zipimporter):
-        pass
-    elif hasattr(sys, 'frozen'):
-        pass
-    else:
-        err = 'It looks like you installed yt-dlp with a package manager, pip, setup.py or a tarball. Please use that to update'
+    ERRORS = {
+        'exe': None,
+        'zip': None,
+        'dir': 'Auto-update is not supported for unpackaged windows executable. Re-download the latest release',
+        'source': 'You cannot update when running from source code',
+        'unknown': 'It looks like you installed yt-dlp with a package manager, pip, setup.py or a tarball. Use that to update',
+    }
+    err = ERRORS.get(detect_variant(), ERRORS['unknown'])
    if err:
        return report_error(err, expected=True)

@ -138,12 +151,7 @@ def run_update(ydl):
        if not urlh:
            return None
        hash_data = ydl._opener.open(urlh).read().decode('utf-8')
-        if hash_data.startswith('version:'):
-            # Old colon-separated hash file
-            return dict(ln.split(':') for ln in hash_data.splitlines()).get(filename)
-        else:
-            # GNU-style hash file
-            return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename)
+        return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename)

    if not os.access(filename, os.W_OK):
        return report_error('no write permissions on %s' % filename, expected=True)