diff --git a/yt_dlp/extractor/bilibili.py b/yt_dlp/extractor/bilibili.py
index a775aa97f..1bbf7ca1c 100644
--- a/yt_dlp/extractor/bilibili.py
+++ b/yt_dlp/extractor/bilibili.py
@@ -225,10 +225,6 @@ class BiliBiliIE(InfoExtractor):
                         'quality': -2 if 'hd.mp4' in backup_url else -3,
                     })
 
-                for a_format in formats:
-                    a_format.setdefault('http_headers', {}).update({
-                        'Referer': url,
-                    })
             for audio in audios:
                 formats.append({
                     'url': audio.get('baseUrl') or audio.get('base_url') or audio.get('url'),
@@ -252,6 +248,9 @@ class BiliBiliIE(InfoExtractor):
                 'id': video_id,
                 'duration': float_or_none(durl.get('length'), 1000),
                 'formats': formats,
+                'http_headers': {
+                    'Referer': url,
+                },
             })
             break