Alex Willmer
3 months ago
committed by
GitHub
commit
ced63f96d9
@ -0,0 +1,13 @@
|
||||
# shellcheck shell=bash
|
||||
|
||||
# Tox environment name -> Python executable name (e.g. py312-m_mtg -> python3.12)
|
||||
toxenv-python() {
|
||||
local pattern='^py([23])([0-9]{1,2}).*'
|
||||
if [[ $1 =~ $pattern ]]; then
|
||||
echo "python${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"
|
||||
return
|
||||
else
|
||||
echo "${FUNCNAME[0]}: $1: environment name not recognised" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env bash
|
||||
set -o errexit -o nounset -o pipefail
|
||||
|
||||
INDENT=" "
|
||||
POSSIBLE_PYTHONS=(
|
||||
python
|
||||
python2
|
||||
python3
|
||||
/usr/bin/python
|
||||
/usr/bin/python2
|
||||
/usr/bin/python3
|
||||
# GitHub macOS 12 images: python2.7 is installed, but not on $PATH
|
||||
/Library/Frameworks/Python.framework/Versions/2.7/bin/python2.7
|
||||
)
|
||||
|
||||
for p in "${POSSIBLE_PYTHONS[@]}"; do
|
||||
echo "$p"
|
||||
if [[ ${p:0:1} == "/" && -e $p ]]; then
|
||||
:
|
||||
elif type "$p" > /dev/null 2>&1; then
|
||||
type "$p" 2>&1 | sed -e "s/^/${INDENT}type: /"
|
||||
else
|
||||
echo "${INDENT}Not present"
|
||||
echo
|
||||
continue
|
||||
fi
|
||||
|
||||
$p -c "import sys; print('${INDENT}version: %d.%d.%d' % sys.version_info[:3])"
|
||||
# macOS builders lack a realpath command
|
||||
$p -c "import os.path; print('${INDENT}realpath: %s' % os.path.realpath('$(type -p "$p")'))"
|
||||
$p -c "import sys; print('${INDENT}sys.executable: %s' % sys.executable)"
|
||||
echo
|
||||
done
|
||||
@ -0,0 +1,36 @@
|
||||
# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: MIT
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 6):
|
||||
from mitogen.imports._py36 import _code_imports
|
||||
elif sys.version_info >= (2, 5):
|
||||
from mitogen.imports._py2 import _code_imports_py25 as _code_imports
|
||||
else:
|
||||
from mitogen.imports._py2 import _code_imports_py24 as _code_imports
|
||||
|
||||
|
||||
def codeobj_imports(co):
|
||||
"""
|
||||
Yield (level, modname, names) tuples by scanning the code object `co`.
|
||||
|
||||
Top level `import mod` & `from mod import foo` statements are matched.
|
||||
Those inside a `class ...` or `def ...` block are currently skipped.
|
||||
|
||||
>>> co = compile('import a, b; from c import d, e as f', '<str>', 'exec')
|
||||
>>> list(codeobj_imports(co)) # doctest: +ELLIPSIS
|
||||
[(..., 'a', ()), (..., 'b', ()), (..., 'c', ('d', 'e'))]
|
||||
|
||||
:return:
|
||||
Generator producing `(level, modname, names)` tuples, where:
|
||||
|
||||
* `level`:
|
||||
-1 implicit relative (Python 2.x default)
|
||||
0 absolute (Python 3.x, `from __future__ import absolute_import`)
|
||||
>0 explicit relative (`from . import a`, `from ..b, import c`)
|
||||
* `modname`: Name of module to import, or to import `names` from.
|
||||
* `names`: tuple of names in `from mod import ..`.
|
||||
"""
|
||||
return _code_imports(co.co_code, co.co_consts, co.co_names)
|
||||
@ -0,0 +1,54 @@
|
||||
# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: MIT
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import array
|
||||
import itertools
|
||||
import opcode
|
||||
|
||||
|
||||
IMPORT_NAME = opcode.opmap['IMPORT_NAME']
|
||||
LOAD_CONST = opcode.opmap['LOAD_CONST']
|
||||
|
||||
|
||||
def _opargs(code, _have_arg=opcode.HAVE_ARGUMENT):
|
||||
it = iter(array.array('B', code))
|
||||
nexti = it.next
|
||||
for i in it:
|
||||
if i >= _have_arg:
|
||||
yield (i, nexti() | (nexti() << 8))
|
||||
else:
|
||||
yield (i, None)
|
||||
|
||||
|
||||
def _code_imports_py25(code, consts, names):
|
||||
it1, it2, it3 = itertools.tee(_opargs(code), 3)
|
||||
try:
|
||||
next(it2)
|
||||
next(it3)
|
||||
next(it3)
|
||||
except StopIteration:
|
||||
return
|
||||
for oparg1, oparg2, (op3, arg3) in itertools.izip(it1, it2, it3):
|
||||
if op3 != IMPORT_NAME:
|
||||
continue
|
||||
op1, arg1 = oparg1
|
||||
op2, arg2 = oparg2
|
||||
if op1 != LOAD_CONST or op2 != LOAD_CONST:
|
||||
continue
|
||||
yield (consts[arg1], names[arg3], consts[arg2] or ())
|
||||
|
||||
|
||||
def _code_imports_py24(code, consts, names):
|
||||
it1, it2 = itertools.tee(_opargs(code), 2)
|
||||
try:
|
||||
next(it2)
|
||||
except StopIteration:
|
||||
return
|
||||
for oparg1, (op2, arg2) in itertools.izip(it1, it2):
|
||||
if op2 != IMPORT_NAME:
|
||||
continue
|
||||
op1, arg1 = oparg1
|
||||
if op1 != LOAD_CONST:
|
||||
continue
|
||||
yield (-1, names[arg2], consts[arg1] or ())
|
||||
@ -0,0 +1,25 @@
|
||||
# SPDX-FileCopyrightText: 2025 Mitogen authors <https://github.com/mitogen-hq>
|
||||
# SPDX-License-Identifier: MIT
|
||||
# !mitogen: minify_safe
|
||||
|
||||
import opcode
|
||||
|
||||
IMPORT_NAME = opcode.opmap['IMPORT_NAME']
|
||||
LOAD_CONST = opcode.opmap['LOAD_CONST']
|
||||
|
||||
|
||||
def _code_imports(code, consts, names):
|
||||
start = 4
|
||||
while True:
|
||||
op3_idx = code.find(IMPORT_NAME, start, -1)
|
||||
if op3_idx < 0:
|
||||
return
|
||||
if op3_idx % 2:
|
||||
start = op3_idx + 1
|
||||
continue
|
||||
if code[op3_idx-4] != LOAD_CONST or code[op3_idx-2] != LOAD_CONST:
|
||||
start = op3_idx + 2
|
||||
continue
|
||||
start = op3_idx + 6
|
||||
arg1, arg2, arg3 = code[op3_idx-3], code[op3_idx-1], code[op3_idx+1]
|
||||
yield (consts[arg1], names[arg3], consts[arg2] or ())
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
set -o nounset
|
||||
|
||||
BENCH_FUNC="codeobj_imports"
|
||||
BENCH_EXPR="deque($BENCH_FUNC(co), maxlen=0)"
|
||||
BIG_MODULE_PATH="$(dirname -- "$0")/data/big_module.py"
|
||||
IMPORTS="from collections import deque; from mitogen.imports import $BENCH_FUNC"
|
||||
COMPILE="co=compile(open('$BIG_MODULE_PATH').read(), '$BIG_MODULE_PATH', 'exec')"
|
||||
PYTHONS=(
|
||||
python2.7 python3.9 python3.10 python3.11 python3.12 python3.13
|
||||
)
|
||||
for p in "${PYTHONS[@]}"; do
|
||||
echo -e -n "$BENCH_FUNC $p "
|
||||
$p -m timeit -s "$IMPORTS" -s "$COMPILE" "$BENCH_EXPR"
|
||||
done
|
||||
@ -0,0 +1,11 @@
|
||||
# pyright: reportMissingImports=false
|
||||
# ruff: noqa: E401 E702 F401 F403
|
||||
|
||||
import a
|
||||
import a.b
|
||||
import c as d
|
||||
import e, e.f as g \
|
||||
, h; import i
|
||||
|
||||
from j import k, l, m as n
|
||||
from o import *
|
||||
@ -0,0 +1,9 @@
|
||||
# pyright: reportMissingImports=false
|
||||
# ruff: noqa: E401 E702 F401 F403
|
||||
|
||||
from . import a
|
||||
from .b import c, d as e
|
||||
from ... import (
|
||||
f,
|
||||
j as k,
|
||||
)
|
||||
@ -0,0 +1,13 @@
|
||||
# pyright: reportMissingImports=false
|
||||
# ruff: noqa: E401 E702 F401 F403
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import a
|
||||
import a.b
|
||||
import c as d
|
||||
import e, e.f as g \
|
||||
, h; import i
|
||||
|
||||
from j import k, l, m as n
|
||||
from o import *
|
||||
@ -0,0 +1,7 @@
|
||||
class C:
|
||||
import in_class
|
||||
from in_class import x as y
|
||||
|
||||
def m(self):
|
||||
import in_method
|
||||
from in_method import x as y, z
|
||||
@ -0,0 +1,3 @@
|
||||
def f():
|
||||
import in_func
|
||||
from in_func import x as y, z
|
||||
@ -0,0 +1,16 @@
|
||||
import sys
|
||||
|
||||
|
||||
if True:
|
||||
import in_if_always_true
|
||||
from in_if_always_true import x as y, z
|
||||
else:
|
||||
import in_else_never_true
|
||||
from in_else_never_true import x as y, z
|
||||
|
||||
if sys.version >= (3, 0):
|
||||
import in_if_py3
|
||||
from in_if_py3 import x as y, z
|
||||
else:
|
||||
import in_else_py2
|
||||
from in_else_py2 import x as y, z
|
||||
@ -0,0 +1,9 @@
|
||||
try:
|
||||
import in_try
|
||||
from in_try import x as y, z
|
||||
except ImportError:
|
||||
import in_except_importerror
|
||||
from in_except_importerror import x as y, z
|
||||
except Exception:
|
||||
import in_except_exception
|
||||
from in_except_exception import x as y, z
|
||||
@ -0,0 +1,2 @@
|
||||
user_policies_max_failed_logins: 10
|
||||
user_policies_users: []
|
||||
@ -0,0 +1,11 @@
|
||||
- name: Set login attempts (macOS)
|
||||
vars:
|
||||
max_failed_logins: "{{ item.policies.max_failed_logins | default(user_policies_max_failed_logins) }}"
|
||||
command: >
|
||||
pwpolicy
|
||||
-u '{{ item.name }}'
|
||||
-setpolicy 'maxFailedLoginAttempts={{ max_failed_logins }}'
|
||||
with_items: "{{ user_policies_users }}"
|
||||
when:
|
||||
- ansible_system == 'Darwin'
|
||||
changed_when: true
|
||||
@ -0,0 +1,130 @@
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
import mitogen.imports
|
||||
|
||||
import testlib
|
||||
|
||||
|
||||
def testmod_compile(path):
|
||||
path = os.path.join(testlib.MODS_DIR, path)
|
||||
f = open(path, 'rb')
|
||||
co = compile(f.read(), path, 'exec')
|
||||
f.close()
|
||||
return co
|
||||
|
||||
|
||||
class ScanCodeImportsTest(testlib.TestCase):
|
||||
func = staticmethod(mitogen.imports.codeobj_imports)
|
||||
|
||||
@unittest.skipIf(sys.version_info < (3, 0), "Py is 2.x, would be relative")
|
||||
def test_default_absolute(self):
|
||||
co = testmod_compile('scanning/defaults.py')
|
||||
expected = [
|
||||
(0, 'a', ()), (0, 'a.b', ()), (0, 'c', ()),
|
||||
(0, 'e', ()), (0, 'e.f', ()), (0, 'h', ()),
|
||||
(0, 'i', ()),
|
||||
(0, 'j', ('k', 'l', 'm')),
|
||||
(0, 'o', ('*',)),
|
||||
]
|
||||
self.assertEqual(list(self.func(co)), expected)
|
||||
|
||||
@unittest.skipIf(sys.version_info >= (3, 0), "Py is 3.x, would be absolute")
|
||||
def test_default_relative(self):
|
||||
co = testmod_compile('scanning/defaults.py')
|
||||
expected = [
|
||||
(-1, 'a', ()), (-1, 'a.b', ()), (-1, 'c', ()),
|
||||
(-1, 'e', ()), (-1, 'e.f', ()), (-1, 'h', ()),
|
||||
(-1, 'i', ()),
|
||||
(-1, 'j', ('k', 'l', 'm')),
|
||||
(-1, 'o', ('*',)),
|
||||
]
|
||||
self.assertEqual(list(self.func(co)), expected)
|
||||
|
||||
@unittest.skipIf(sys.version_info < (2, 5), "Py is 2.4, no absolute_import")
|
||||
def test_explicit_absolute(self):
|
||||
co = testmod_compile('scanning/has_absolute_import.py')
|
||||
expected = [
|
||||
(0, '__future__', ('absolute_import',)),
|
||||
|
||||
(0, 'a', ()), (0, 'a.b', ()), (0, 'c', ()),
|
||||
(0, 'e', ()), (0, 'e.f', ()), (0, 'h', ()),
|
||||
(0, 'i', ()),
|
||||
(0, 'j', ('k', 'l', 'm')),
|
||||
(0, 'o', ('*',)),
|
||||
]
|
||||
self.assertEqual(list(self.func(co)), expected)
|
||||
|
||||
@unittest.skipIf(sys.version_info < (2, 5), "Py is 2.4, no `from . import x`")
|
||||
def test_explicit_relative(self):
|
||||
co = testmod_compile('scanning/explicit_relative.py')
|
||||
expected = [
|
||||
(1, '', ('a',)),
|
||||
(1, 'b', ('c', 'd')),
|
||||
(3, '', ('f', 'j')),
|
||||
]
|
||||
self.assertEqual(list(self.func(co)), expected)
|
||||
|
||||
def test_scoped_class(self):
|
||||
# Imports in `class` or `def` are ignored, a bad heuristc to detect
|
||||
# lazy imports and skip sending the pre-emptively.
|
||||
# See
|
||||
# - https://github.com/mitogen-hq/mitogen/issues/682
|
||||
# - https://github.com/mitogen-hq/mitogen/issues/1325#issuecomment-3170482014
|
||||
co = testmod_compile('scanning/scoped_class.py')
|
||||
self.assertEqual(list(self.func(co)), [])
|
||||
|
||||
pass
|
||||
|
||||
def test_scoped_function(self):
|
||||
co = testmod_compile('scanning/scoped_function.py')
|
||||
self.assertEqual(list(self.func(co)), [])
|
||||
|
||||
@unittest.skipIf(sys.version_info >= (3, 0), "Python is 3.x, which prunes")
|
||||
def test_scoped_if_else_unpruned(self):
|
||||
co = testmod_compile('scanning/scoped_if_else.py')
|
||||
level = (-1, 0)[int(sys.version_info >= (3, 0))]
|
||||
expected = [
|
||||
(level, 'sys', ()),
|
||||
(level, 'in_if_always_true', ()),
|
||||
(level, 'in_if_always_true', ('x', 'z')),
|
||||
# Python 2.x does no pruning
|
||||
(level, 'in_else_never_true', ()),
|
||||
(level, 'in_else_never_true', ('x', 'z')),
|
||||
(level, 'in_if_py3', ()),
|
||||
(level, 'in_if_py3', ('x', 'z')),
|
||||
(level, 'in_else_py2', ()),
|
||||
(level, 'in_else_py2', ('x', 'z')),
|
||||
]
|
||||
self.assertEqual(list(self.func(co)), expected)
|
||||
|
||||
@unittest.skipIf(sys.version_info < (3, 0), "Python is 2.x, which doesn't prune")
|
||||
def test_scoped_if_else_pruned(self):
|
||||
co = testmod_compile('scanning/scoped_if_else.py')
|
||||
level = (-1, 0)[int(sys.version_info >= (3, 0))]
|
||||
expected = [
|
||||
(level, 'sys', ()),
|
||||
(level, 'in_if_always_true', ()),
|
||||
(level, 'in_if_always_true', ('x', 'z')),
|
||||
# Python 3.x prunes some impossible branches ...
|
||||
(level, 'in_if_py3', ()),
|
||||
(level, 'in_if_py3', ('x', 'z')),
|
||||
# ... but not sys.version_info ones
|
||||
(level, 'in_else_py2', ()),
|
||||
(level, 'in_else_py2', ('x', 'z')),
|
||||
]
|
||||
self.assertEqual(list(self.func(co)), expected)
|
||||
|
||||
def test_scoped_try_except(self):
|
||||
co = testmod_compile('scanning/scoped_try_except.py')
|
||||
level = (-1, 0)[int(sys.version_info >= (3, 0))]
|
||||
expected = [
|
||||
(level, 'in_try', ()),
|
||||
(level, 'in_try', ('x', 'z')),
|
||||
(level, 'in_except_importerror', ()),
|
||||
(level, 'in_except_importerror', ('x', 'z')),
|
||||
(level, 'in_except_exception', ()),
|
||||
(level, 'in_except_exception', ('x', 'z')),
|
||||
]
|
||||
self.assertEqual(list(self.func(co)), expected)
|
||||
@ -1,25 +0,0 @@
|
||||
import inspect
|
||||
|
||||
import testlib
|
||||
import mitogen.master
|
||||
|
||||
|
||||
class ScanCodeImportsTest(testlib.TestCase):
|
||||
func = staticmethod(mitogen.master.scan_code_imports)
|
||||
|
||||
if mitogen.core.PY3:
|
||||
level = 0
|
||||
else:
|
||||
level = -1
|
||||
|
||||
SIMPLE_EXPECT = [
|
||||
(level, 'inspect', ()),
|
||||
(level, 'testlib', ()),
|
||||
(level, 'mitogen.master', ()),
|
||||
]
|
||||
|
||||
def test_simple(self):
|
||||
source_path = inspect.getsourcefile(ScanCodeImportsTest)
|
||||
with open(source_path) as f:
|
||||
co = compile(f.read(), source_path, 'exec')
|
||||
self.assertEqual(list(self.func(co)), self.SIMPLE_EXPECT)
|
||||
Loading…
Reference in New Issue