|
|
|
|
@ -32,67 +32,11 @@ from ..utils.traversal import require, traverse_obj
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TwitterBaseIE(InfoExtractor):
|
|
|
|
|
_NETRC_MACHINE = 'twitter'
|
|
|
|
|
_API_BASE = 'https://api.x.com/1.1/'
|
|
|
|
|
_GRAPHQL_API_BASE = 'https://x.com/i/api/graphql/'
|
|
|
|
|
_BASE_REGEX = r'https?://(?:(?:www|m(?:obile)?)\.)?(?:(?:twitter|x)\.com|twitter3e4tixl4xyajtrzo62zg5vztmjuricljdp2c5kshju4avyoid\.onion)/'
|
|
|
|
|
_AUTH = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
|
|
|
|
|
_LEGACY_AUTH = 'AAAAAAAAAAAAAAAAAAAAAIK1zgAAAAAA2tUWuhGZ2JceoId5GwYWU5GspY4%3DUq7gzFoCZs1QfwGoVdvSac3IniczZEYXIcDyumCauIXpcAPorE'
|
|
|
|
|
_flow_token = None
|
|
|
|
|
|
|
|
|
|
_LOGIN_INIT_DATA = json.dumps({
|
|
|
|
|
'input_flow_data': {
|
|
|
|
|
'flow_context': {
|
|
|
|
|
'debug_overrides': {},
|
|
|
|
|
'start_location': {
|
|
|
|
|
'location': 'unknown',
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
'subtask_versions': {
|
|
|
|
|
'action_list': 2,
|
|
|
|
|
'alert_dialog': 1,
|
|
|
|
|
'app_download_cta': 1,
|
|
|
|
|
'check_logged_in_account': 1,
|
|
|
|
|
'choice_selection': 3,
|
|
|
|
|
'contacts_live_sync_permission_prompt': 0,
|
|
|
|
|
'cta': 7,
|
|
|
|
|
'email_verification': 2,
|
|
|
|
|
'end_flow': 1,
|
|
|
|
|
'enter_date': 1,
|
|
|
|
|
'enter_email': 2,
|
|
|
|
|
'enter_password': 5,
|
|
|
|
|
'enter_phone': 2,
|
|
|
|
|
'enter_recaptcha': 1,
|
|
|
|
|
'enter_text': 5,
|
|
|
|
|
'enter_username': 2,
|
|
|
|
|
'generic_urt': 3,
|
|
|
|
|
'in_app_notification': 1,
|
|
|
|
|
'interest_picker': 3,
|
|
|
|
|
'js_instrumentation': 1,
|
|
|
|
|
'menu_dialog': 1,
|
|
|
|
|
'notifications_permission_prompt': 2,
|
|
|
|
|
'open_account': 2,
|
|
|
|
|
'open_home_timeline': 1,
|
|
|
|
|
'open_link': 1,
|
|
|
|
|
'phone_verification': 4,
|
|
|
|
|
'privacy_options': 1,
|
|
|
|
|
'security_key': 3,
|
|
|
|
|
'select_avatar': 4,
|
|
|
|
|
'select_banner': 2,
|
|
|
|
|
'settings_list': 7,
|
|
|
|
|
'show_code': 1,
|
|
|
|
|
'sign_up': 2,
|
|
|
|
|
'sign_up_review': 4,
|
|
|
|
|
'tweet_selection_urt': 1,
|
|
|
|
|
'update_users': 1,
|
|
|
|
|
'upload_media': 1,
|
|
|
|
|
'user_recommendations_list': 4,
|
|
|
|
|
'user_recommendations_urt': 1,
|
|
|
|
|
'wait_spinner': 3,
|
|
|
|
|
'web_modal': 1,
|
|
|
|
|
},
|
|
|
|
|
}, separators=(',', ':')).encode()
|
|
|
|
|
|
|
|
|
|
def _extract_variant_formats(self, variant, video_id):
|
|
|
|
|
variant_url = variant.get('url')
|
|
|
|
|
@ -172,135 +116,6 @@ class TwitterBaseIE(InfoExtractor):
|
|
|
|
|
'x-csrf-token': try_call(lambda: self._get_cookies(self._API_BASE)['ct0'].value),
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
def _call_login_api(self, note, headers, query={}, data=None):
|
|
|
|
|
response = self._download_json(
|
|
|
|
|
f'{self._API_BASE}onboarding/task.json', None, note,
|
|
|
|
|
headers=headers, query=query, data=data, expected_status=400)
|
|
|
|
|
error = traverse_obj(response, ('errors', 0, 'message', {str}))
|
|
|
|
|
if error:
|
|
|
|
|
raise ExtractorError(f'Login failed, Twitter API says: {error}', expected=True)
|
|
|
|
|
elif traverse_obj(response, 'status') != 'success':
|
|
|
|
|
raise ExtractorError('Login was unsuccessful')
|
|
|
|
|
|
|
|
|
|
subtask = traverse_obj(
|
|
|
|
|
response, ('subtasks', ..., 'subtask_id', {str}), get_all=False)
|
|
|
|
|
if not subtask:
|
|
|
|
|
raise ExtractorError('Twitter API did not return next login subtask')
|
|
|
|
|
|
|
|
|
|
self._flow_token = response['flow_token']
|
|
|
|
|
|
|
|
|
|
return subtask
|
|
|
|
|
|
|
|
|
|
def _perform_login(self, username, password):
|
|
|
|
|
if self.is_logged_in:
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
guest_token = self._fetch_guest_token(None)
|
|
|
|
|
headers = {
|
|
|
|
|
**self._set_base_headers(),
|
|
|
|
|
'content-type': 'application/json',
|
|
|
|
|
'x-guest-token': guest_token,
|
|
|
|
|
'x-twitter-client-language': 'en',
|
|
|
|
|
'x-twitter-active-user': 'yes',
|
|
|
|
|
'Referer': 'https://x.com/',
|
|
|
|
|
'Origin': 'https://x.com',
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def build_login_json(*subtask_inputs):
|
|
|
|
|
return json.dumps({
|
|
|
|
|
'flow_token': self._flow_token,
|
|
|
|
|
'subtask_inputs': subtask_inputs,
|
|
|
|
|
}, separators=(',', ':')).encode()
|
|
|
|
|
|
|
|
|
|
def input_dict(subtask_id, text):
|
|
|
|
|
return {
|
|
|
|
|
'subtask_id': subtask_id,
|
|
|
|
|
'enter_text': {
|
|
|
|
|
'text': text,
|
|
|
|
|
'link': 'next_link',
|
|
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
next_subtask = self._call_login_api(
|
|
|
|
|
'Downloading flow token', headers, query={'flow_name': 'login'}, data=self._LOGIN_INIT_DATA)
|
|
|
|
|
|
|
|
|
|
while not self.is_logged_in:
|
|
|
|
|
if next_subtask == 'LoginJsInstrumentationSubtask':
|
|
|
|
|
next_subtask = self._call_login_api(
|
|
|
|
|
'Submitting JS instrumentation response', headers, data=build_login_json({
|
|
|
|
|
'subtask_id': next_subtask,
|
|
|
|
|
'js_instrumentation': {
|
|
|
|
|
'response': '{}',
|
|
|
|
|
'link': 'next_link',
|
|
|
|
|
},
|
|
|
|
|
}))
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'LoginEnterUserIdentifierSSO':
|
|
|
|
|
next_subtask = self._call_login_api(
|
|
|
|
|
'Submitting username', headers, data=build_login_json({
|
|
|
|
|
'subtask_id': next_subtask,
|
|
|
|
|
'settings_list': {
|
|
|
|
|
'setting_responses': [{
|
|
|
|
|
'key': 'user_identifier',
|
|
|
|
|
'response_data': {
|
|
|
|
|
'text_data': {
|
|
|
|
|
'result': username,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
}],
|
|
|
|
|
'link': 'next_link',
|
|
|
|
|
},
|
|
|
|
|
}))
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'LoginEnterAlternateIdentifierSubtask':
|
|
|
|
|
next_subtask = self._call_login_api(
|
|
|
|
|
'Submitting alternate identifier', headers,
|
|
|
|
|
data=build_login_json(input_dict(next_subtask, self._get_tfa_info(
|
|
|
|
|
'one of username, phone number or email that was not used as --username'))))
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'LoginEnterPassword':
|
|
|
|
|
next_subtask = self._call_login_api(
|
|
|
|
|
'Submitting password', headers, data=build_login_json({
|
|
|
|
|
'subtask_id': next_subtask,
|
|
|
|
|
'enter_password': {
|
|
|
|
|
'password': password,
|
|
|
|
|
'link': 'next_link',
|
|
|
|
|
},
|
|
|
|
|
}))
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'AccountDuplicationCheck':
|
|
|
|
|
next_subtask = self._call_login_api(
|
|
|
|
|
'Submitting account duplication check', headers, data=build_login_json({
|
|
|
|
|
'subtask_id': next_subtask,
|
|
|
|
|
'check_logged_in_account': {
|
|
|
|
|
'link': 'AccountDuplicationCheck_false',
|
|
|
|
|
},
|
|
|
|
|
}))
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'LoginTwoFactorAuthChallenge':
|
|
|
|
|
next_subtask = self._call_login_api(
|
|
|
|
|
'Submitting 2FA token', headers, data=build_login_json(input_dict(
|
|
|
|
|
next_subtask, self._get_tfa_info('two-factor authentication token'))))
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'LoginAcid':
|
|
|
|
|
next_subtask = self._call_login_api(
|
|
|
|
|
'Submitting confirmation code', headers, data=build_login_json(input_dict(
|
|
|
|
|
next_subtask, self._get_tfa_info('confirmation code sent to your email or phone'))))
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'ArkoseLogin':
|
|
|
|
|
self.raise_login_required('Twitter is requiring captcha for this login attempt', method='cookies')
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'DenyLoginSubtask':
|
|
|
|
|
self.raise_login_required('Twitter rejected this login attempt as suspicious', method='cookies')
|
|
|
|
|
|
|
|
|
|
elif next_subtask == 'LoginSuccessSubtask':
|
|
|
|
|
raise ExtractorError('Twitter API did not grant auth token cookie')
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
|
raise ExtractorError(f'Unrecognized subtask ID "{next_subtask}"')
|
|
|
|
|
|
|
|
|
|
self.report_login()
|
|
|
|
|
|
|
|
|
|
def _call_api(self, path, video_id, query={}, graphql=False):
|
|
|
|
|
headers = self._set_base_headers(legacy=not graphql and self._selected_api == 'legacy')
|
|
|
|
|
headers.update({
|
|
|
|
|
|