From b5899f4f19116bb4d98907413fa3fb84a952ef13 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 22 May 2022 17:07:18 +0530 Subject: [PATCH] [build, cleanup] Refactor Closes #3835, #3837 --- .github/workflows/build.yml | 444 ++++++++++++++++-------------------- Makefile | 5 +- pyinst.py | 49 ++-- yt_dlp/update.py | 248 +++++++++----------- yt_dlp/utils.py | 4 +- 5 files changed, 340 insertions(+), 410 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6820889e2..bb9507165 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,27 +2,20 @@ name: Build on: workflow_dispatch jobs: - build_unix: + create_release: runs-on: ubuntu-latest outputs: version_suffix: ${{ steps.version_suffix.outputs.version_suffix }} ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }} upload_url: ${{ steps.create_release.outputs.upload_url }} - sha256_bin: ${{ steps.sha256_bin.outputs.sha256_bin }} - sha512_bin: ${{ steps.sha512_bin.outputs.sha512_bin }} - sha256_tar: ${{ steps.sha256_tar.outputs.sha256_tar }} - sha512_tar: ${{ steps.sha512_tar.outputs.sha512_tar }} - steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up Python - uses: actions/setup-python@v2 + - uses: actions/setup-python@v2 with: - python-version: '3.8' - - name: Install packages - run: sudo apt-get -y install zip pandoc man + python-version: '3.10' + - name: Set version suffix id: version_suffix env: @@ -34,83 +27,27 @@ jobs: run: | python devscripts/update-version.py ${{ steps.version_suffix.outputs.version_suffix }} make issuetemplates + - name: Push to release - id: push_release run: | git config --global user.name github-actions git config --global user.email github-actions@example.com git add -u - git commit -m "[version] update" -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all" + git commit -m "[version] update" -m "Created by: ${{ github.event.sender.login }}" -m ":ci skip all :ci run dl" git push origin --force ${{ github.event.ref }}:release echo ::set-output name=head_sha::$(git rev-parse HEAD) - name: Update master - id: push_master env: PUSH_VERSION_COMMIT: ${{ secrets.PUSH_VERSION_COMMIT }} if: "env.PUSH_VERSION_COMMIT != ''" run: git push origin ${{ github.event.ref }} - name: Get Changelog - id: get_changelog run: | changelog=$(cat Changelog.md | grep -oPz '(?s)(?<=### ${{ steps.bump_version.outputs.ytdlp_version }}\n{2}).+?(?=\n{2,3}###)') || true echo "changelog<> $GITHUB_ENV echo "$changelog" >> $GITHUB_ENV echo "EOF" >> $GITHUB_ENV - - name: Build lazy extractors - id: lazy_extractors - run: python devscripts/make_lazy_extractors.py - - name: Run Make - run: make all tar - - name: Get SHA2-256SUMS for yt-dlp - id: sha256_bin - run: echo "::set-output name=sha256_bin::$(sha256sum yt-dlp | awk '{print $1}')" - - name: Get SHA2-256SUMS for yt-dlp.tar.gz - id: sha256_tar - run: echo "::set-output name=sha256_tar::$(sha256sum yt-dlp.tar.gz | awk '{print $1}')" - - name: Get SHA2-512SUMS for yt-dlp - id: sha512_bin - run: echo "::set-output name=sha512_bin::$(sha512sum yt-dlp | awk '{print $1}')" - - name: Get SHA2-512SUMS for yt-dlp.tar.gz - id: sha512_tar - run: echo "::set-output name=sha512_tar::$(sha512sum yt-dlp.tar.gz | awk '{print $1}')" - - - name: Install dependencies for pypi - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - if: "env.PYPI_TOKEN != ''" - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish on pypi - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - if: "env.TWINE_PASSWORD != ''" - run: | - rm -rf dist/* - python setup.py sdist bdist_wheel - twine upload dist/* - - - name: Install SSH private key - env: - BREW_TOKEN: ${{ secrets.BREW_TOKEN }} - if: "env.BREW_TOKEN != ''" - uses: yt-dlp/ssh-agent@v0.5.3 - with: - ssh-private-key: ${{ env.BREW_TOKEN }} - - name: Update Homebrew Formulae - env: - BREW_TOKEN: ${{ secrets.BREW_TOKEN }} - if: "env.BREW_TOKEN != ''" - run: | - git clone git@github.com:yt-dlp/homebrew-taps taps/ - python3 devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ steps.bump_version.outputs.ytdlp_version }}" - git -C taps/ config user.name github-actions - git -C taps/ config user.email github-actions@example.com - git -C taps/ commit -am 'yt-dlp: ${{ steps.bump_version.outputs.ytdlp_version }}' - git -C taps/ push - - name: Create Release id: create_release uses: actions/create-release@v1 @@ -129,13 +66,51 @@ jobs: ${{ env.changelog }} draft: false prerelease: false - - name: Upload yt-dlp Unix binary - id: upload-release-asset + + + build_unix: + needs: create_release + runs-on: ubuntu-latest + outputs: + sha256_bin: ${{ steps.get_sha.outputs.sha256_bin }} + sha512_bin: ${{ steps.get_sha.outputs.sha512_bin }} + sha256_tar: ${{ steps.get_sha.outputs.sha256_tar }} + sha512_tar: ${{ steps.get_sha.outputs.sha512_tar }} + sha256_unix_zip: ${{ steps.get_sha.outputs.sha256_unix_zip }} + sha512_unix_zip: ${{ steps.get_sha.outputs.sha512_unix_zip }} + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.10' + - name: Install Requirements + run: | + sudo apt-get -y install zip pandoc man + python -m pip install --upgrade pip setuptools wheel twine + python -m pip install Pyinstaller -r requirements.txt + + - name: Prepare + run: | + python devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }} + python devscripts/make_lazy_extractors.py + - name: Build UNIX executables + run: | + make all tar + - name: Get SHA2-SUMS + id: get_sha + run: | + echo "::set-output name=sha256_bin::$(sha256sum yt-dlp | awk '{print $1}')" + echo "::set-output name=sha512_bin::$(sha512sum yt-dlp | awk '{print $1}')" + echo "::set-output name=sha256_tar::$(sha256sum yt-dlp.tar.gz | awk '{print $1}')" + echo "::set-output name=sha512_tar::$(sha512sum yt-dlp.tar.gz | awk '{print $1}')" + + - name: Upload zip binary uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.create_release.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./yt-dlp asset_name: yt-dlp asset_content_type: application/octet-stream @@ -144,270 +119,247 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ steps.create_release.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./yt-dlp.tar.gz asset_name: yt-dlp.tar.gz asset_content_type: application/gzip + - name: Build and publish on PyPi + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + if: "env.TWINE_PASSWORD != ''" + run: | + rm -rf dist/* + python setup.py sdist bdist_wheel + twine upload dist/* + + - name: Install SSH private key for Homebrew + env: + BREW_TOKEN: ${{ secrets.BREW_TOKEN }} + if: "env.BREW_TOKEN != ''" + uses: yt-dlp/ssh-agent@v0.5.3 + with: + ssh-private-key: ${{ env.BREW_TOKEN }} + - name: Update Homebrew Formulae + env: + BREW_TOKEN: ${{ secrets.BREW_TOKEN }} + if: "env.BREW_TOKEN != ''" + run: | + git clone git@github.com:yt-dlp/homebrew-taps taps/ + python devscripts/update-formulae.py taps/Formula/yt-dlp.rb "${{ steps.bump_version.outputs.ytdlp_version }}" + git -C taps/ config user.name github-actions + git -C taps/ config user.email github-actions@example.com + git -C taps/ commit -am 'yt-dlp: ${{ steps.bump_version.outputs.ytdlp_version }}' + git -C taps/ push + + build_macos: runs-on: macos-11 - needs: build_unix + needs: create_release outputs: - sha256_macos: ${{ steps.sha256_macos.outputs.sha256_macos }} - sha512_macos: ${{ steps.sha512_macos.outputs.sha512_macos }} - sha256_macos_zip: ${{ steps.sha256_macos_zip.outputs.sha256_macos_zip }} - sha512_macos_zip: ${{ steps.sha512_macos_zip.outputs.sha512_macos_zip }} + sha256_macos: ${{ steps.get_sha.outputs.sha256_macos }} + sha512_macos: ${{ steps.get_sha.outputs.sha512_macos }} + sha256_macos_zip: ${{ steps.get_sha.outputs.sha256_macos_zip }} + sha512_macos_zip: ${{ steps.get_sha.outputs.sha512_macos_zip }} steps: - uses: actions/checkout@v2 - # In order to create a universal2 application, the version of python3 in /usr/bin has to be used + # NB: In order to create a universal2 application, the version of python3 in /usr/bin has to be used - name: Install Requirements run: | brew install coreutils - /usr/bin/python3 -m pip install -U --user pip Pyinstaller==4.10 -r requirements.txt - - name: Bump version - id: bump_version - run: /usr/bin/python3 devscripts/update-version.py - - name: Build lazy extractors - id: lazy_extractors - run: /usr/bin/python3 devscripts/make_lazy_extractors.py - - name: Run PyInstaller Script - run: /usr/bin/python3 pyinst.py --target-architecture universal2 --onefile - - name: Upload yt-dlp MacOS binary - id: upload-release-macos + /usr/bin/python3 -m pip install -U --user pip Pyinstaller -r requirements.txt + + - name: Prepare + run: | + /usr/bin/python3 devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }} + /usr/bin/python3 devscripts/make_lazy_extractors.py + - name: Build + run: | + /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir + (cd ./dist/yt-dlp_macos && zip -r ../yt-dlp_macos.zip .) + /usr/bin/python3 pyinst.py --target-architecture universal2 + - name: Get SHA2-SUMS + id: get_sha + run: | + echo "::set-output name=sha256_macos::$(sha256sum dist/yt-dlp_macos | awk '{print $1}')" + echo "::set-output name=sha512_macos::$(sha512sum dist/yt-dlp_macos | awk '{print $1}')" + echo "::set-output name=sha256_macos_zip::$(sha256sum dist/yt-dlp_macos.zip | awk '{print $1}')" + echo "::set-output name=sha512_macos_zip::$(sha512sum dist/yt-dlp_macos.zip | awk '{print $1}')" + + - name: Upload standalone binary uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./dist/yt-dlp_macos asset_name: yt-dlp_macos asset_content_type: application/octet-stream - - name: Get SHA2-256SUMS for yt-dlp_macos - id: sha256_macos - run: echo "::set-output name=sha256_macos::$(sha256sum dist/yt-dlp_macos | awk '{print $1}')" - - name: Get SHA2-512SUMS for yt-dlp_macos - id: sha512_macos - run: echo "::set-output name=sha512_macos::$(sha512sum dist/yt-dlp_macos | awk '{print $1}')" - - - name: Run PyInstaller Script with --onedir - run: | - /usr/bin/python3 pyinst.py --target-architecture universal2 --onedir - zip ./dist/yt-dlp_macos.zip ./dist/yt-dlp_macos - - name: Upload yt-dlp MacOS onedir - id: upload-release-macos-zip + - name: Upload onedir binary uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./dist/yt-dlp_macos.zip asset_name: yt-dlp_macos.zip asset_content_type: application/zip - - name: Get SHA2-256SUMS for yt-dlp_macos.zip - id: sha256_macos_zip - run: echo "::set-output name=sha256_macos_zip::$(sha256sum dist/yt-dlp_macos.zip | awk '{print $1}')" - - name: Get SHA2-512SUMS for yt-dlp_macos.zip - id: sha512_macos_zip - run: echo "::set-output name=sha512_macos_zip::$(sha512sum dist/yt-dlp_macos.zip | awk '{print $1}')" + build_windows: runs-on: windows-latest - needs: build_unix + needs: create_release outputs: - sha256_win: ${{ steps.sha256_win.outputs.sha256_win }} - sha512_win: ${{ steps.sha512_win.outputs.sha512_win }} - sha256_py2exe: ${{ steps.sha256_py2exe.outputs.sha256_py2exe }} - sha512_py2exe: ${{ steps.sha512_py2exe.outputs.sha512_py2exe }} - sha256_win_zip: ${{ steps.sha256_win_zip.outputs.sha256_win_zip }} - sha512_win_zip: ${{ steps.sha512_win_zip.outputs.sha512_win_zip }} + sha256_win: ${{ steps.get_sha.outputs.sha256_win }} + sha512_win: ${{ steps.get_sha.outputs.sha512_win }} + sha256_py2exe: ${{ steps.get_sha.outputs.sha256_py2exe }} + sha512_py2exe: ${{ steps.get_sha.outputs.sha512_py2exe }} + sha256_win_zip: ${{ steps.get_sha.outputs.sha256_win_zip }} + sha512_win_zip: ${{ steps.get_sha.outputs.sha512_win_zip }} steps: - uses: actions/checkout@v2 - # 3.8 is used for Win7 support - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: + - uses: actions/setup-python@v2 + with: # 3.8 is used for Win7 support python-version: '3.8' - name: Install Requirements - # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds - run: | + run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds python -m pip install --upgrade pip setuptools wheel py2exe pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.10-py3-none-any.whl" -r requirements.txt - - name: Bump version - id: bump_version + + - name: Prepare + run: | + python devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }} + python devscripts/make_lazy_extractors.py + - name: Build + run: | + python setup.py py2exe + Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe + python pyinst.py + python pyinst.py --onedir + Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip + - name: Get SHA2-SUMS + id: get_sha + run: | + echo "::set-output name=sha256_py2exe::$((Get-FileHash dist\yt-dlp_min.exe -Algorithm SHA256).Hash.ToLower())" + echo "::set-output name=sha512_py2exe::$((Get-FileHash dist\yt-dlp_min.exe -Algorithm SHA512).Hash.ToLower())" + echo "::set-output name=sha256_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" + echo "::set-output name=sha512_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())" + echo "::set-output name=sha256_win_zip::$((Get-FileHash dist\yt-dlp_win.zip -Algorithm SHA256).Hash.ToLower())" + echo "::set-output name=sha512_win_zip::$((Get-FileHash dist\yt-dlp_win.zip -Algorithm SHA512).Hash.ToLower())" + + - name: Upload py2exe binary + uses: actions/upload-release-asset@v1 env: - version_suffix: ${{ needs.build_unix.outputs.version_suffix }} - run: python devscripts/update-version.py ${{ env.version_suffix }} - - name: Build lazy extractors - id: lazy_extractors - run: python devscripts/make_lazy_extractors.py - - name: Run PyInstaller Script - run: python pyinst.py - - name: Upload yt-dlp.exe Windows binary - id: upload-release-windows + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./dist/yt-dlp_min.exe + asset_name: yt-dlp_min.exe + asset_content_type: application/vnd.microsoft.portable-executable + - name: Upload standalone binary uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./dist/yt-dlp.exe asset_name: yt-dlp.exe asset_content_type: application/vnd.microsoft.portable-executable - - name: Get SHA2-256SUMS for yt-dlp.exe - id: sha256_win - run: echo "::set-output name=sha256_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" - - name: Get SHA2-512SUMS for yt-dlp.exe - id: sha512_win - run: echo "::set-output name=sha512_win::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())" - - - name: Run PyInstaller Script with --onedir - run: | - python pyinst.py --onedir - Compress-Archive -LiteralPath ./dist/yt-dlp -DestinationPath ./dist/yt-dlp_win.zip - - name: Upload yt-dlp Windows onedir - id: upload-release-windows-zip + - name: Upload onedir binary uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./dist/yt-dlp_win.zip asset_name: yt-dlp_win.zip asset_content_type: application/zip - - name: Get SHA2-256SUMS for yt-dlp_win.zip - id: sha256_win_zip - run: echo "::set-output name=sha256_win_zip::$((Get-FileHash dist\yt-dlp_win.zip -Algorithm SHA256).Hash.ToLower())" - - name: Get SHA2-512SUMS for yt-dlp_win.zip - id: sha512_win_zip - run: echo "::set-output name=sha512_win_zip::$((Get-FileHash dist\yt-dlp_win.zip -Algorithm SHA512).Hash.ToLower())" - - - name: Run py2exe Script - run: python setup.py py2exe - - name: Upload yt-dlp_min.exe Windows binary - id: upload-release-windows-py2exe - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} - asset_path: ./dist/yt-dlp.exe - asset_name: yt-dlp_min.exe - asset_content_type: application/vnd.microsoft.portable-executable - - name: Get SHA2-256SUMS for yt-dlp_min.exe - id: sha256_py2exe - run: echo "::set-output name=sha256_py2exe::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" - - name: Get SHA2-512SUMS for yt-dlp_min.exe - id: sha512_py2exe - run: echo "::set-output name=sha512_py2exe::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())" + build_windows32: runs-on: windows-latest - needs: build_unix - + needs: create_release outputs: - sha256_win32: ${{ steps.sha256_win32.outputs.sha256_win32 }} - sha512_win32: ${{ steps.sha512_win32.outputs.sha512_win32 }} + sha256_win32: ${{ steps.get_sha.outputs.sha256_win32 }} + sha512_win32: ${{ steps.get_sha.outputs.sha512_win32 }} steps: - uses: actions/checkout@v2 - # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390 - - name: Set up Python 3.7 32-Bit - uses: actions/setup-python@v2 - with: + - uses: actions/setup-python@v2 + with: # 3.7 is used for Vista support. See https://github.com/yt-dlp/yt-dlp/issues/390 python-version: '3.7' architecture: 'x86' - name: Install Requirements run: | python -m pip install --upgrade pip setuptools wheel pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.10-py3-none-any.whl" -r requirements.txt - - name: Bump version - id: bump_version - env: - version_suffix: ${{ needs.build_unix.outputs.version_suffix }} - run: python devscripts/update-version.py ${{ env.version_suffix }} - - name: Build lazy extractors - id: lazy_extractors - run: python devscripts/make_lazy_extractors.py - - name: Run PyInstaller Script for 32 Bit - run: python pyinst.py - - name: Upload Executable yt-dlp_x86.exe - id: upload-release-windows32 + + - name: Prepare + run: | + python devscripts/update-version.py ${{ needs.create_release.outputs.version_suffix }} + python devscripts/make_lazy_extractors.py + - name: Build + run: | + python pyinst.py + - name: Get SHA2-SUMS + id: get_sha + run: | + echo "::set-output name=sha256_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())" + echo "::set-output name=sha512_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())" + + - name: Upload standalone binary uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./dist/yt-dlp_x86.exe asset_name: yt-dlp_x86.exe asset_content_type: application/vnd.microsoft.portable-executable - - name: Get SHA2-256SUMS for yt-dlp_x86.exe - id: sha256_win32 - run: echo "::set-output name=sha256_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())" - - name: Get SHA2-512SUMS for yt-dlp_x86.exe - id: sha512_win32 - run: echo "::set-output name=sha512_win32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())" + finish: runs-on: ubuntu-latest - needs: [build_unix, build_windows, build_windows32, build_macos] + needs: [create_release, build_unix, build_windows, build_windows32, build_macos] steps: - - name: Make SHA2-256SUMS file - env: - SHA256_BIN: ${{ needs.build_unix.outputs.sha256_bin }} - SHA256_TAR: ${{ needs.build_unix.outputs.sha256_tar }} - SHA256_WIN: ${{ needs.build_windows.outputs.sha256_win }} - SHA256_PY2EXE: ${{ needs.build_windows.outputs.sha256_py2exe }} - SHA256_WIN_ZIP: ${{ needs.build_windows.outputs.sha256_win_zip }} - SHA256_WIN32: ${{ needs.build_windows32.outputs.sha256_win32 }} - SHA256_MACOS: ${{ needs.build_macos.outputs.sha256_macos }} - SHA256_MACOS_ZIP: ${{ needs.build_macos.outputs.sha256_macos_zip }} + - name: Make SHA2-SUMS files run: | - echo "${{ env.SHA256_BIN }} yt-dlp" >> SHA2-256SUMS - echo "${{ env.SHA256_TAR }} yt-dlp.tar.gz" >> SHA2-256SUMS - echo "${{ env.SHA256_WIN }} yt-dlp.exe" >> SHA2-256SUMS - echo "${{ env.SHA256_PY2EXE }} yt-dlp_min.exe" >> SHA2-256SUMS - echo "${{ env.SHA256_WIN32 }} yt-dlp_x86.exe" >> SHA2-256SUMS - echo "${{ env.SHA256_WIN_ZIP }} yt-dlp_win.zip" >> SHA2-256SUMS - echo "${{ env.SHA256_MACOS }} yt-dlp_macos" >> SHA2-256SUMS - echo "${{ env.SHA256_MACOS_ZIP }} yt-dlp_macos.zip" >> SHA2-256SUMS - - name: Upload 256SUMS file - id: upload-sums + echo "${{ needs.build_unix.outputs.sha256_bin }} yt-dlp" >> SHA2-256SUMS + echo "${{ needs.build_unix.outputs.sha256_tar }} yt-dlp.tar.gz" >> SHA2-256SUMS + echo "${{ needs.build_windows.outputs.sha256_win }} yt-dlp.exe" >> SHA2-256SUMS + echo "${{ needs.build_windows.outputs.sha256_py2exe }} yt-dlp_min.exe" >> SHA2-256SUMS + echo "${{ needs.build_windows32.outputs.sha256_win32 }} yt-dlp_x86.exe" >> SHA2-256SUMS + echo "${{ needs.build_windows.outputs.sha256_win_zip }} yt-dlp_win.zip" >> SHA2-256SUMS + echo "${{ needs.build_macos.outputs.sha256_macos }} yt-dlp_macos" >> SHA2-256SUMS + echo "${{ needs.build_macos.outputs.sha256_macos_zip }} yt-dlp_macos.zip" >> SHA2-256SUMS + echo "${{ needs.build_unix.outputs.sha512_bin }} yt-dlp" >> SHA2-512SUMS + echo "${{ needs.build_unix.outputs.sha512_tar }} yt-dlp.tar.gz" >> SHA2-512SUMS + echo "${{ needs.build_windows.outputs.sha512_win }} yt-dlp.exe" >> SHA2-512SUMS + echo "${{ needs.build_windows.outputs.sha512_py2exe }} yt-dlp_min.exe" >> SHA2-512SUMS + echo "${{ needs.build_windows32.outputs.sha512_win32 }} yt-dlp_x86.exe" >> SHA2-512SUMS + echo "${{ needs.build_windows.outputs.sha512_win_zip }} yt-dlp_win.zip" >> SHA2-512SUMS + echo "${{ needs.build_macos.outputs.sha512_macos }} yt-dlp_macos" >> SHA2-512SUMS + echo "${{ needs.build_macos.outputs.sha512_macos_zip }} yt-dlp_macos.zip" >> SHA2-512SUMS + + - name: Upload SHA2-256SUMS file uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./SHA2-256SUMS asset_name: SHA2-256SUMS asset_content_type: text/plain - - name: Make SHA2-512SUMS file - env: - SHA512_BIN: ${{ needs.build_unix.outputs.sha512_bin }} - SHA512_TAR: ${{ needs.build_unix.outputs.sha512_tar }} - SHA512_WIN: ${{ needs.build_windows.outputs.sha512_win }} - SHA512_PY2EXE: ${{ needs.build_windows.outputs.sha512_py2exe }} - SHA512_WIN_ZIP: ${{ needs.build_windows.outputs.sha512_win_zip }} - SHA512_WIN32: ${{ needs.build_windows32.outputs.sha512_win32 }} - SHA512_MACOS: ${{ needs.build_macos.outputs.sha512_macos }} - SHA512_MACOS_ZIP: ${{ needs.build_macos.outputs.sha512_macos_zip }} - run: | - echo "${{ env.SHA512_BIN }} yt-dlp" >> SHA2-512SUMS - echo "${{ env.SHA512_TAR }} yt-dlp.tar.gz" >> SHA2-512SUMS - echo "${{ env.SHA512_WIN }} yt-dlp.exe" >> SHA2-512SUMS - echo "${{ env.SHA512_WIN_ZIP }} yt-dlp_win.zip" >> SHA2-512SUMS - echo "${{ env.SHA512_PY2EXE }} yt-dlp_min.exe" >> SHA2-512SUMS - echo "${{ env.SHA512_WIN32 }} yt-dlp_x86.exe" >> SHA2-512SUMS - echo "${{ env.SHA512_MACOS }} yt-dlp_macos" >> SHA2-512SUMS - echo "${{ env.SHA512_MACOS_ZIP }} yt-dlp_macos.zip" >> SHA2-512SUMS - - name: Upload 512SUMS file - id: upload-512sums + - name: Upload SHA2-512SUMS file uses: actions/upload-release-asset@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - upload_url: ${{ needs.build_unix.outputs.upload_url }} + upload_url: ${{ needs.create_release.outputs.upload_url }} asset_path: ./SHA2-512SUMS asset_name: SHA2-512SUMS asset_content_type: text/plain diff --git a/Makefile b/Makefile index 3e5885c1d..f12282edc 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,8 @@ tar: yt-dlp.tar.gz # Keep this list in sync with MANIFEST.in # intended use: when building a source distribution, # make pypi-files && python setup.py sdist -pypi-files: - AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \ - completions yt-dlp.1 requirements.txt devscripts/* test/* +pypi-files: AUTHORS Changelog.md LICENSE README.md README.txt supportedsites \ + completions yt-dlp.1 requirements.txt setup.cfg devscripts/* test/* .PHONY: all clean install test tar pypi-files completions ot offlinetest codetest supportedsites diff --git a/pyinst.py b/pyinst.py index af80c1812..de3504b35 100644 --- a/pyinst.py +++ b/pyinst.py @@ -5,24 +5,8 @@ import sys from PyInstaller.__main__ import run as run_pyinstaller -OS_NAME = platform.system() -if OS_NAME == 'Windows': - from PyInstaller.utils.win32.versioninfo import ( - FixedFileInfo, - SetVersion, - StringFileInfo, - StringStruct, - StringTable, - VarFileInfo, - VarStruct, - VSVersionInfo, - ) -elif OS_NAME == 'Darwin': - pass -else: - raise Exception(f'{OS_NAME} is not supported') -ARCH = platform.architecture()[0][:2] +OS_NAME, ARCH = sys.platform, platform.architecture()[0][:2] def main(): @@ -33,10 +17,7 @@ def main(): if not onedir and '-F' not in opts and '--onefile' not in opts: opts.append('--onefile') - name = 'yt-dlp%s' % ('_macos' if OS_NAME == 'Darwin' else '_x86' if ARCH == '32' else '') - final_file = ''.join(( - 'dist/', f'{name}/' if onedir else '', name, '.exe' if OS_NAME == 'Windows' else '')) - + name, final_file = exe(onedir) print(f'Building yt-dlp v{version} {ARCH}bit for {OS_NAME} with options {opts}') print('Remember to update the version using "devscripts/update-version.py"') if not os.path.isfile('yt_dlp/extractor/lazy_extractors.py'): @@ -79,6 +60,21 @@ def read_version(fname): return locals()['__version__'] +def exe(onedir): + """@returns (name, path)""" + name = '_'.join(filter(None, ( + 'yt-dlp', + OS_NAME == 'darwin' and 'macos', + ARCH == '32' and 'x86' + ))) + return name, ''.join(filter(None, ( + 'dist/', + onedir and f'{name}/', + name, + OS_NAME == 'win32' and '.exe' + ))) + + def version_to_list(version): version_list = version.split('.') return list(map(int, version_list)) + [0] * (4 - len(version_list)) @@ -114,6 +110,17 @@ def set_version_info(exe, version): def windows_set_version(exe, version): + from PyInstaller.utils.win32.versioninfo import ( + FixedFileInfo, + SetVersion, + StringFileInfo, + StringStruct, + StringTable, + VarFileInfo, + VarStruct, + VSVersionInfo, + ) + version_list = version_to_list(version) suffix = '_x86' if ARCH == '32' else '' SetVersion(exe, VSVersionInfo( diff --git a/yt_dlp/update.py b/yt_dlp/update.py index 861e2495b..3de7c7209 100644 --- a/yt_dlp/update.py +++ b/yt_dlp/update.py @@ -4,27 +4,29 @@ import os import platform import subprocess import sys -import traceback from zipimport import zipimporter -from .compat import compat_realpath, functools -from .utils import Popen, encode_compat_str, write_string +from .compat import functools # isort: split +from .compat import compat_realpath +from .utils import Popen, traverse_obj, version_tuple from .version import __version__ +RELEASE_JSON_URL = 'https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest' + + @functools.cache -def get_variant_and_executable_path(): +def _get_variant_and_executable_path(): """@returns (variant, executable_path)""" if hasattr(sys, 'frozen'): path = sys.executable - prefix = 'mac' if sys.platform == 'darwin' else 'win' - if getattr(sys, '_MEIPASS', None): - if sys._MEIPASS == os.path.dirname(sys.executable): - return f'{prefix}_dir', path - return f'{prefix}_exe', path - return 'py2exe', path - - path = os.path.join(os.path.dirname(__file__), '..') + if not hasattr(sys, '_MEIPASS'): + return 'py2exe', path + if sys._MEIPASS == os.path.dirname(path): + return f'{sys.platform}_dir', path + return f'{sys.platform}_exe', path + + path = os.path.dirname(__file__) if isinstance(__loader__, zipimporter): return 'zip', os.path.join(path, '..') elif os.path.basename(sys.argv[0]) == '__main__.py': @@ -33,23 +35,28 @@ def get_variant_and_executable_path(): def detect_variant(): - return get_variant_and_executable_path()[0] + return _get_variant_and_executable_path()[0] +_FILE_SUFFIXES = { + 'zip': '', + 'py2exe': '_min.exe', + 'win32_exe': '.exe', + 'darwin_exe': '_macos', +} + _NON_UPDATEABLE_REASONS = { - 'win_exe': None, - 'zip': None, - 'mac_exe': None, - 'py2exe': None, - 'win_dir': 'Auto-update is not supported for unpackaged windows executable; Re-download the latest release', - 'mac_dir': 'Auto-update is not supported for unpackaged MacOS executable; Re-download the latest release', + **{variant: None for variant in _FILE_SUFFIXES}, # Updatable + **{variant: f'Auto-update is not supported for unpackaged {name} executable; Re-download the latest release' + for variant, name in {'win32_dir': 'Windows', 'darwin_dir': 'MacOS'}.items()}, 'source': 'You cannot update when running from source code; Use git to pull the latest changes', 'unknown': 'It looks like you installed yt-dlp with a package manager, pip or setup.py; Use that to update', + 'other': 'It looks like you are using an unofficial build of yt-dlp; Build the executable again', } def is_non_updateable(): - return _NON_UPDATEABLE_REASONS.get(detect_variant(), _NON_UPDATEABLE_REASONS['unknown']) + return _NON_UPDATEABLE_REASONS.get(detect_variant(), _NON_UPDATEABLE_REASONS['other']) def run_update(ydl): @@ -58,8 +65,6 @@ def run_update(ydl): Returns whether the program should terminate """ - JSON_URL = 'https://api.github.com/repos/yt-dlp/yt-dlp/releases/latest' - def report_error(msg, expected=False): ydl.report_error(msg, tb=False if expected else None) @@ -74,23 +79,17 @@ def run_update(ydl): def calc_sha256sum(path): h = hashlib.sha256() - b = bytearray(128 * 1024) - mv = memoryview(b) + mv = memoryview(bytearray(128 * 1024)) with open(os.path.realpath(path), 'rb', buffering=0) as f: for n in iter(lambda: f.readinto(mv), 0): h.update(mv[:n]) return h.hexdigest() - # Download and check versions info try: - version_info = ydl._opener.open(JSON_URL).read().decode() - version_info = json.loads(version_info) + version_info = json.loads(ydl.urlopen(RELEASE_JSON_URL).read().decode()) except Exception: return report_network_error('obtain version info', delim='; Please try again later or') - def version_tuple(version_str): - return tuple(map(int, version_str.split('.'))) - version_id = version_info['tag_name'] ydl.to_screen(f'Latest version: {version_id}, Current version: {__version__}') if version_tuple(__version__) >= version_tuple(version_id): @@ -101,157 +100,130 @@ def run_update(ydl): if err: return report_error(err, True) - variant, filename = get_variant_and_executable_path() + variant, filename = _get_variant_and_executable_path() filename = compat_realpath(filename) # Absolute path, following symlinks + label = _FILE_SUFFIXES[variant] + if label and platform.architecture()[0][:2] == '32': + label = f'_x86{label}' + release_name = f'yt-dlp{label}' + ydl.to_screen(f'Current Build Hash {calc_sha256sum(filename)}') ydl.to_screen(f'Updating to version {version_id} ...') - version_labels = { - 'zip_3': '', - 'win_exe_64': '.exe', - 'py2exe_64': '_min.exe', - 'win_exe_32': '_x86.exe', - 'mac_exe_64': '_macos', - } - - def get_bin_info(bin_or_exe, version): - label = version_labels[f'{bin_or_exe}_{version}'] - return next((i for i in version_info['assets'] if i['name'] == 'yt-dlp%s' % label), {}) - - def get_sha256sum(bin_or_exe, version): - filename = 'yt-dlp%s' % version_labels[f'{bin_or_exe}_{version}'] - urlh = next( - (i for i in version_info['assets'] if i['name'] in ('SHA2-256SUMS')), - {}).get('browser_download_url') - if not urlh: - return None - hash_data = ydl._opener.open(urlh).read().decode() - return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename) - - if not os.access(filename, os.W_OK): - return report_permission_error(filename) - - if variant in ('win_exe', 'py2exe'): - directory = os.path.dirname(filename) - if not os.access(directory, os.W_OK): - return report_permission_error(directory) + def get_file(name, fatal=True): + error = report_network_error if fatal else lambda _: None + url = traverse_obj( + version_info, ('assets', lambda _, v: v['name'] == name, 'browser_download_url'), get_all=False) + if not url: + return error('fetch updates') try: - if os.path.exists(filename + '.old'): - os.remove(filename + '.old') + return ydl.urlopen(url).read() except OSError: - return report_unable('remove the old version') - - try: - arch = platform.architecture()[0][:2] - url = get_bin_info(variant, arch).get('browser_download_url') - if not url: - return report_network_error('fetch updates') - urlh = ydl._opener.open(url) - newcontent = urlh.read() - urlh.close() - except OSError: - return report_network_error('download latest version') - - try: - with open(filename + '.new', 'wb') as outf: - outf.write(newcontent) - except OSError: - return report_permission_error(f'{filename}.new') - - expected_sum = get_sha256sum(variant, arch) - if not expected_sum: + return error('download latest version') + + def verify(content): + if not content: + return False + hash_data = get_file('SHA2-256SUMS', fatal=False) or b'' + expected = dict(ln.split()[::-1] for ln in hash_data.decode().splitlines()).get(release_name) + if not expected: ydl.report_warning('no hash information found for the release') - elif calc_sha256sum(filename + '.new') != expected_sum: - report_network_error('verify the new executable') - try: - os.remove(filename + '.new') - except OSError: - return report_unable('remove corrupt download') + elif hashlib.sha256(content).hexdigest() != expected: + return report_network_error('verify the new executable') + return True - try: - os.rename(filename, filename + '.old') - except OSError: - return report_unable('move current version') - try: - os.rename(filename + '.new', filename) - except OSError: - report_unable('overwrite current version') - os.rename(filename + '.old', filename) - return - try: - # Continues to run in the background - Popen( - 'ping 127.0.0.1 -n 5 -w 1000 & del /F "%s.old"' % filename, - shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - ydl.to_screen('Updated yt-dlp to version %s' % version_id) - return True # Exit app - except OSError: - report_unable('delete the old version') + directory = os.path.dirname(filename) + if not os.access(filename, os.W_OK): + return report_permission_error(filename) + elif not os.access(directory, os.W_OK): + return report_permission_error(directory) - elif variant in ('zip', 'mac_exe'): - pack_type = '3' if variant == 'zip' else '64' - try: - url = get_bin_info(variant, pack_type).get('browser_download_url') - if not url: - return report_network_error('fetch updates') - urlh = ydl._opener.open(url) - newcontent = urlh.read() - urlh.close() - except OSError: - return report_network_error('download the latest version') + new_filename, old_filename = f'{filename}.new', f'{filename}.old' + if variant == 'zip': # Can be replaced in-place + new_filename, old_filename = filename, None - expected_sum = get_sha256sum(variant, pack_type) - if not expected_sum: - ydl.report_warning('no hash information found for the release') - elif hashlib.sha256(newcontent).hexdigest() != expected_sum: - return report_network_error('verify the new package') + try: + if os.path.exists(old_filename or ''): + os.remove(old_filename) + except OSError: + return report_unable('remove the old version') - try: - with open(filename, 'wb') as outf: - outf.write(newcontent) - except OSError: - return report_unable('overwrite current version') + newcontent = get_file(release_name) + if not verify(newcontent): + return + try: + with open(new_filename, 'wb') as outf: + outf.write(newcontent) + except OSError: + return report_permission_error(new_filename) + + try: + if old_filename: + os.rename(filename, old_filename) + except OSError: + return report_unable('move current version') + try: + if old_filename: + os.rename(new_filename, filename) + except OSError: + report_unable('overwrite current version') + os.rename(old_filename, filename) + return - ydl.to_screen('Updated yt-dlp to version %s; Restart yt-dlp to use the new version' % version_id) + if variant not in ('win32_exe', 'py2exe'): + if old_filename: + os.remove(old_filename) + ydl.to_screen(f'Updated yt-dlp to version {version_id}; Restart yt-dlp to use the new version') return - assert False, f'Unhandled variant: {variant}' + try: + # Continues to run in the background + Popen(f'ping 127.0.0.1 -n 5 -w 1000 & del /F "{old_filename}"', + shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + ydl.to_screen(f'Updated yt-dlp to version {version_id}') + return True # Exit app + except OSError: + report_unable('delete the old version') # Deprecated def update_self(to_screen, verbose, opener): - - printfn = to_screen + import traceback + from .utils import write_string write_string( 'DeprecationWarning: "yt_dlp.update.update_self" is deprecated and may be removed in a future version. ' 'Use "yt_dlp.update.run_update(ydl)" instead\n') + printfn = to_screen + class FakeYDL(): - _opener = opener to_screen = printfn @staticmethod def report_warning(msg, *args, **kwargs): - return printfn('WARNING: %s' % msg, *args, **kwargs) + return printfn(f'WARNING: {msg}', *args, **kwargs) @staticmethod def report_error(msg, tb=None): - printfn('ERROR: %s' % msg) + printfn(f'ERROR: {msg}') if not verbose: return if tb is None: - # Copied from YoutubeDl.trouble + # Copied from YoutubeDL.trouble if sys.exc_info()[0]: tb = '' if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info)) - tb += encode_compat_str(traceback.format_exc()) + tb += traceback.format_exc() else: tb_data = traceback.format_list(traceback.extract_stack()) tb = ''.join(tb_data) if tb: printfn(tb) + def urlopen(self, url): + return opener.open(url) + return run_update(FakeYDL()) diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 12204433d..2e3c51562 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -4904,9 +4904,9 @@ def make_dir(path, to_screen=None): def get_executable_path(): - from .update import get_variant_and_executable_path + from .update import _get_variant_and_executable_path - return os.path.abspath(get_variant_and_executable_path()[1]) + return os.path.dirname(os.path.abspath(_get_variant_and_executable_path()[1])) def load_plugins(name, suffix, namespace):