archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

yum - only instantiate YumBase once (#63713)

Browse Source 
* yum - only instantiate YumBase once

Previously, this code was re-instantiating the `YumBase` object
many times which is unnecessary and slow. However, we must do it
twice in the `state: absent` case because the `yumSack` and
`rpmSack` data of the previously instantiated object becomes
invalid and is no longer useful post transaction when we verify
that the package removal did in fact take place. Also, this patch
removes the repetitive re-processing of enable/disable of repos in
various places.

Here's a display of the speed increase against a RHEL7 host:

```yaml
- hosts: rhel7
  remote_user: root
  tasks:
  - name: Install generic packages
    yum:
      state: present
      name:
        - iptraf-ng
        - screen
        - erlang
  - name: Remove generic packages
    yum:
      state: absent
      name:
        - iptraf-ng
        - screen
        - erlang
```

Before this patch:
```
real    0m52.728s
user    0m5.645s
sys     0m0.482s
```

After this patch:
```
real    0m17.139s
user    0m3.238s
sys     0m0.277s
```

Fixes #63588
Fixes #63551

Signed-off-by: Adam Miller <admiller@redhat.com>

* add changelog

Signed-off-by: Adam Miller <admiller@redhat.com>
pull/66383/head
Adam Miller 5 years ago committed by Matt Clay
parent 13c40c70db
commit 610487df3c
2 changed files with 81 additions and 86 deletions
Show Stats Download Patch File Download Diff File

4
changelogs/fragments/63551-yum-single-YumBase-instantiation.yaml
Unescape Escape View File

@ -0,0 +1,4 @@
bugfixes:
- yum - performance bugfix, the YumBase object was being instantiated
multiple times unnecessarily, which lead to considerable overhead when
operating against large sets of packages.

163
lib/ansible/modules/packaging/os/yum.py
Unescape Escape View File

@ -391,6 +391,7 @@ class YumModule(YumDnf):
self.pkg_mgr_name = "yum"
self.lockfile = '/var/run/yum.pid'
self._yum_base = None
def _enablerepos_with_error_checking(self, yumbase):
# NOTE: This seems unintuitive, but it mirrors yum's CLI behavior
@ -454,34 +455,51 @@ class YumModule(YumDnf):
# another copy seems to be running
return True
@property
def yum_base(self):
my = yum.YumBase()
my.preconf.debuglevel = 0
my.preconf.errorlevel = 0
my.preconf.plugins = True
my.preconf.enabled_plugins = self.enable_plugin
my.preconf.disabled_plugins = self.disable_plugin
if self.releasever:
my.preconf.releasever = self.releasever
if self.installroot != '/':
# do not setup installroot by default, because of error
# CRITICAL:yum.cli:Config Error: Error accessing file for config file:////etc/yum.conf
# in old yum version (like in CentOS 6.6)
my.preconf.root = self.installroot
my.conf.installroot = self.installroot
if self.conf_file and os.path.exists(self.conf_file):
my.preconf.fn = self.conf_file
if os.geteuid() != 0:
if hasattr(my, 'setCacheDir'):
my.setCacheDir()
else:
cachedir = yum.misc.getCacheDir()
my.repos.setCacheDir(cachedir)
my.conf.cache = 0
if self.disable_excludes:
my.conf.disable_excludes = self.disable_excludes
if self._yum_base:
return self._yum_base
else:
# Only init once
self._yum_base = yum.YumBase()
self._yum_base.preconf.debuglevel = 0
self._yum_base.preconf.errorlevel = 0
self._yum_base.preconf.plugins = True
self._yum_base.preconf.enabled_plugins = self.enable_plugin
self._yum_base.preconf.disabled_plugins = self.disable_plugin
if self.releasever:
self._yum_base.preconf.releasever = self.releasever
if self.installroot != '/':
# do not setup installroot by default, because of error
# CRITICAL:yum.cli:Config Error: Error accessing file for config file:////etc/yum.conf
# in old yum version (like in CentOS 6.6)
self._yum_base.preconf.root = self.installroot
self._yum_base.conf.installroot = self.installroot
if self.conf_file and os.path.exists(self.conf_file):
self._yum_base.preconf.fn = self.conf_file
if os.geteuid() != 0:
if hasattr(self._yum_base, 'setCacheDir'):
self._yum_base.setCacheDir()
else:
cachedir = yum.misc.getCacheDir()
self._yum_base.repos.setCacheDir(cachedir)
self._yum_base.conf.cache = 0
if self.disable_excludes:
self._yum_base.conf.disable_excludes = self.disable_excludes
return my
# A sideeffect of accessing conf is that the configuration is
# loaded and plugins are discovered
self.yum_base.conf
try:
self._enablerepos_with_error_checking(self._yum_base)
for rid in self.disablerepo:
self.yum_base.repos.disableRepo(rid)
except Exception as e:
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
return self._yum_base
def po_to_envra(self, po):
if hasattr(po, 'ui_envra'):
@ -492,11 +510,10 @@ class YumModule(YumDnf):
def is_group_env_installed(self, name):
name_lower = name.lower()
my = self.yum_base()
if yum.__version_info__ >= (3, 4):
groups_list = my.doGroupLists(return_evgrps=True)
groups_list = self.yum_base.doGroupLists(return_evgrps=True)
else:
groups_list = my.doGroupLists()
groups_list = self.yum_base.doGroupLists()
# list of the installed groups on the first index
groups = groups_list[0]
@ -520,15 +537,10 @@ class YumModule(YumDnf):
if not repoq:
pkgs = []
try:
my = self.yum_base()
for rid in self.disablerepo:
my.repos.disableRepo(rid)
self._enablerepos_with_error_checking(my)
e, m, _ = my.rpmdb.matchPackageNames([pkgspec])
e, m, _ = self.yum_base.rpmdb.matchPackageNames([pkgspec])
pkgs = e + m
if not pkgs and not is_pkg:
pkgs.extend(my.returnInstalledPackagesByDep(pkgspec))
pkgs.extend(self.yum_base.returnInstalledPackagesByDep(pkgspec))
except Exception as e:
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
@ -574,15 +586,10 @@ class YumModule(YumDnf):
pkgs = []
try:
my = self.yum_base()
for rid in self.disablerepo:
my.repos.disableRepo(rid)
self._enablerepos_with_error_checking(my)
e, m, _ = my.pkgSack.matchPackageNames([pkgspec])
e, m, _ = self.yum_base.pkgSack.matchPackageNames([pkgspec])
pkgs = e + m
if not pkgs:
pkgs.extend(my.returnPackagesByDep(pkgspec))
pkgs.extend(self.yum_base.returnPackagesByDep(pkgspec))
except Exception as e:
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
@ -613,16 +620,12 @@ class YumModule(YumDnf):
updates = []
try:
my = self.yum_base()
for rid in self.disablerepo:
my.repos.disableRepo(rid)
self._enablerepos_with_error_checking(my)
pkgs = my.returnPackagesByDep(pkgspec) + my.returnInstalledPackagesByDep(pkgspec)
pkgs = self.yum_base.returnPackagesByDep(pkgspec) + \
self.yum_base.returnInstalledPackagesByDep(pkgspec)
if not pkgs:
e, m, _ = my.pkgSack.matchPackageNames([pkgspec])
e, m, _ = self.yum_base.pkgSack.matchPackageNames([pkgspec])
pkgs = e + m
updates = my.doPackageLists(pkgnarrow='updates').updates
updates = self.yum_base.doPackageLists(pkgnarrow='updates').updates
except Exception as e:
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
@ -653,13 +656,9 @@ class YumModule(YumDnf):
pkgs = []
try:
my = self.yum_base()
for rid in self.disablerepo:
my.repos.disableRepo(rid)
self._enablerepos_with_error_checking(my)
try:
pkgs = my.returnPackagesByDep(req_spec) + my.returnInstalledPackagesByDep(req_spec)
pkgs = self.yum_base.returnPackagesByDep(req_spec) + \
self.yum_base.returnInstalledPackagesByDep(req_spec)
except Exception as e:
# If a repo with `repo_gpgcheck=1` is added and the repo GPG
# key was never accepted, querying this repo will throw an
@ -668,14 +667,15 @@ class YumModule(YumDnf):
# the key and try again.
if 'repomd.xml signature could not be verified' in to_native(e):
self.module.run_command(self.yum_basecmd + ['makecache'])
pkgs = my.returnPackagesByDep(req_spec) + my.returnInstalledPackagesByDep(req_spec)
pkgs = self.yum_base.returnPackagesByDep(req_spec) + \
self.yum_base.returnInstalledPackagesByDep(req_spec)
else:
raise
if not pkgs:
e, m, _ = my.pkgSack.matchPackageNames([req_spec])
e, m, _ = self.yum_base.pkgSack.matchPackageNames([req_spec])
pkgs.extend(e)
pkgs.extend(m)
e, m, _ = my.rpmdb.matchPackageNames([req_spec])
e, m, _ = self.yum_base.rpmdb.matchPackageNames([req_spec])
pkgs.extend(e)
pkgs.extend(m)
except Exception as e:
@ -767,21 +767,20 @@ class YumModule(YumDnf):
@contextmanager
def set_env_proxy(self):
# setting system proxy environment and saving old, if exists
my = self.yum_base()
namepass = ""
scheme = ["http", "https"]
old_proxy_env = [os.getenv("http_proxy"), os.getenv("https_proxy")]
try:
# "_none_" is a special value to disable proxy in yum.conf/*.repo
if my.conf.proxy and my.conf.proxy not in ("_none_",):
if my.conf.proxy_username:
namepass = namepass + my.conf.proxy_username
proxy_url = my.conf.proxy
if my.conf.proxy_password:
namepass = namepass + ":" + my.conf.proxy_password
elif '@' in my.conf.proxy:
namepass = my.conf.proxy.split('@')[0].split('//')[-1]
proxy_url = my.conf.proxy.replace("{0}@".format(namepass), "")
if self.yum_base.conf.proxy and self.yum_base.conf.proxy not in ("_none_",):
if self.yum_base.conf.proxy_username:
namepass = namepass + self.yum_base.conf.proxy_username
proxy_url = self.yum_base.conf.proxy
if self.yum_base.conf.proxy_password:
namepass = namepass + ":" + self.yum_base.conf.proxy_password
elif '@' in self.yum_base.conf.proxy:
namepass = self.yum_base.conf.proxy.split('@')[0].split('//')[-1]
proxy_url = self.yum_base.conf.proxy.replace("{0}@".format(namepass), "")
if namepass:
namepass = namepass + '@'
@ -792,7 +791,7 @@ class YumModule(YumDnf):
)
else:
for item in scheme:
os.environ[item + "_proxy"] = my.conf.proxy
os.environ[item + "_proxy"] = self.yum_base.conf.proxy
yield
except yum.Errors.YumBaseError:
raise
@ -1139,6 +1138,7 @@ class YumModule(YumDnf):
# of the process
# at this point we check to see if the pkg is no longer present
self._yum_base = None # previous YumBase package index is now invalid
for pkg in pkgs:
if pkg.startswith('@'):
installed = self.is_group_env_installed(pkg)
@ -1471,9 +1471,9 @@ class YumModule(YumDnf):
can be done for remove and absent action
As solution I would advice to cal
try: my.repos.disableRepo(disablerepo)
try: self.yum_base.repos.disableRepo(disablerepo)
and
try: my.repos.enableRepo(enablerepo)
try: self.yum_base.repos.enableRepo(enablerepo)
right before any yum_cmd is actually called regardless
of yum action.
@ -1490,19 +1490,14 @@ class YumModule(YumDnf):
if self.update_cache:
self.module.run_command(self.yum_basecmd + ['clean', 'expire-cache'])
my = self.yum_base()
try:
if self.disablerepo:
for rid in self.disablerepo:
my.repos.disableRepo(rid)
current_repos = my.repos.repos.keys()
current_repos = self.yum_base.repos.repos.keys()
if self.enablerepo:
try:
self._enablerepos_with_error_checking(my)
new_repos = my.repos.repos.keys()
new_repos = self.yum_base.repos.repos.keys()
for i in new_repos:
if i not in current_repos:
rid = my.repos.getRepo(i)
rid = self.yum_base.repos.getRepo(i)
a = rid.repoXML.repoid # nopep8 - https://github.com/ansible/ansible/pull/21475#pullrequestreview-22404868
current_repos = new_repos
except yum.Errors.YumBaseError as e:
@ -1598,13 +1593,9 @@ class YumModule(YumDnf):
# the system then users will see an error message using the yum API.
# Use repoquery in those cases.
my = self.yum_base()
# A sideeffect of accessing conf is that the configuration is
# loaded and plugins are discovered
my.conf
repoquery = None
try:
yum_plugins = my.plugins._plugins
yum_plugins = self.yum_base.plugins._plugins
except AttributeError:
pass
else:

Write Preview
Loading…
Cancel
Save