archive
/
ansible
mirror of https://github.com/ansible/ansible.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

yum - only instantiate YumBase once (#63713)

Browse Source 
* yum - only instantiate YumBase once

Previously, this code was re-instantiating the `YumBase` object
many times which is unnecessary and slow. However, we must do it
twice in the `state: absent` case because the `yumSack` and
`rpmSack` data of the previously instantiated object becomes
invalid and is no longer useful post transaction when we verify
that the package removal did in fact take place. Also, this patch
removes the repetitive re-processing of enable/disable of repos in
various places.

Here's a display of the speed increase against a RHEL7 host:

```yaml
- hosts: rhel7
  remote_user: root
  tasks:
  - name: Install generic packages
    yum:
      state: present
      name:
        - iptraf-ng
        - screen
        - erlang
  - name: Remove generic packages
    yum:
      state: absent
      name:
        - iptraf-ng
        - screen
        - erlang
```

Before this patch:
```
real    0m52.728s
user    0m5.645s
sys     0m0.482s
```

After this patch:
```
real    0m17.139s
user    0m3.238s
sys     0m0.277s
```

Fixes #63588
Fixes #63551

Signed-off-by: Adam Miller <admiller@redhat.com>

* add changelog

Signed-off-by: Adam Miller <admiller@redhat.com>
pull/66383/head
Adam Miller 5 years ago committed by Matt Clay
parent 13c40c70db
commit 610487df3c
2 changed files with 81 additions and 86 deletions
Show Stats Download Patch File Download Diff File

4
changelogs/fragments/63551-yum-single-YumBase-instantiation.yaml
Unescape Escape View File

@ -0,0 +1,4 @@
bugfixes:
- yum - performance bugfix, the YumBase object was being instantiated
multiple times unnecessarily, which lead to considerable overhead when
operating against large sets of packages.

163
lib/ansible/modules/packaging/os/yum.py
Unescape Escape View File

@ -391,6 +391,7 @@ class YumModule(YumDnf):
self.pkg_mgr_name = "yum" self.pkg_mgr_name = "yum"
self.lockfile = '/var/run/yum.pid' self.lockfile = '/var/run/yum.pid'
self._yum_base = None
def _enablerepos_with_error_checking(self, yumbase): def _enablerepos_with_error_checking(self, yumbase):
# NOTE: This seems unintuitive, but it mirrors yum's CLI behavior # NOTE: This seems unintuitive, but it mirrors yum's CLI behavior
@ -454,34 +455,51 @@ class YumModule(YumDnf):
# another copy seems to be running # another copy seems to be running
return True return True
@property
def yum_base(self): def yum_base(self):
my = yum.YumBase() if self._yum_base:
my.preconf.debuglevel = 0 return self._yum_base
my.preconf.errorlevel = 0 else:
my.preconf.plugins = True # Only init once
my.preconf.enabled_plugins = self.enable_plugin self._yum_base = yum.YumBase()
my.preconf.disabled_plugins = self.disable_plugin self._yum_base.preconf.debuglevel = 0
if self.releasever: self._yum_base.preconf.errorlevel = 0
my.preconf.releasever = self.releasever self._yum_base.preconf.plugins = True
if self.installroot != '/': self._yum_base.preconf.enabled_plugins = self.enable_plugin
# do not setup installroot by default, because of error self._yum_base.preconf.disabled_plugins = self.disable_plugin
# CRITICAL:yum.cli:Config Error: Error accessing file for config file:////etc/yum.conf if self.releasever:
# in old yum version (like in CentOS 6.6) self._yum_base.preconf.releasever = self.releasever
my.preconf.root = self.installroot if self.installroot != '/':
my.conf.installroot = self.installroot # do not setup installroot by default, because of error
if self.conf_file and os.path.exists(self.conf_file): # CRITICAL:yum.cli:Config Error: Error accessing file for config file:////etc/yum.conf
my.preconf.fn = self.conf_file # in old yum version (like in CentOS 6.6)
if os.geteuid() != 0: self._yum_base.preconf.root = self.installroot
if hasattr(my, 'setCacheDir'): self._yum_base.conf.installroot = self.installroot
my.setCacheDir() if self.conf_file and os.path.exists(self.conf_file):
else: self._yum_base.preconf.fn = self.conf_file
cachedir = yum.misc.getCacheDir() if os.geteuid() != 0:
my.repos.setCacheDir(cachedir) if hasattr(self._yum_base, 'setCacheDir'):
my.conf.cache = 0 self._yum_base.setCacheDir()
if self.disable_excludes: else:
my.conf.disable_excludes = self.disable_excludes cachedir = yum.misc.getCacheDir()
self._yum_base.repos.setCacheDir(cachedir)
self._yum_base.conf.cache = 0
if self.disable_excludes:
self._yum_base.conf.disable_excludes = self.disable_excludes
return my # A sideeffect of accessing conf is that the configuration is
# loaded and plugins are discovered
self.yum_base.conf
try:
self._enablerepos_with_error_checking(self._yum_base)
for rid in self.disablerepo:
self.yum_base.repos.disableRepo(rid)
except Exception as e:
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
return self._yum_base
def po_to_envra(self, po): def po_to_envra(self, po):
if hasattr(po, 'ui_envra'): if hasattr(po, 'ui_envra'):
@ -492,11 +510,10 @@ class YumModule(YumDnf):
def is_group_env_installed(self, name): def is_group_env_installed(self, name):
name_lower = name.lower() name_lower = name.lower()
my = self.yum_base()
if yum.__version_info__ >= (3, 4): if yum.__version_info__ >= (3, 4):
groups_list = my.doGroupLists(return_evgrps=True) groups_list = self.yum_base.doGroupLists(return_evgrps=True)
else: else:
groups_list = my.doGroupLists() groups_list = self.yum_base.doGroupLists()
# list of the installed groups on the first index # list of the installed groups on the first index
groups = groups_list[0] groups = groups_list[0]
@ -520,15 +537,10 @@ class YumModule(YumDnf):
if not repoq: if not repoq:
pkgs = [] pkgs = []
try: try:
my = self.yum_base() e, m, _ = self.yum_base.rpmdb.matchPackageNames([pkgspec])
for rid in self.disablerepo:
my.repos.disableRepo(rid)
self._enablerepos_with_error_checking(my)
e, m, _ = my.rpmdb.matchPackageNames([pkgspec])
pkgs = e + m pkgs = e + m
if not pkgs and not is_pkg: if not pkgs and not is_pkg:
pkgs.extend(my.returnInstalledPackagesByDep(pkgspec)) pkgs.extend(self.yum_base.returnInstalledPackagesByDep(pkgspec))
except Exception as e: except Exception as e:
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e)) self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
@ -574,15 +586,10 @@ class YumModule(YumDnf):
pkgs = [] pkgs = []
try: try:
my = self.yum_base() e, m, _ = self.yum_base.pkgSack.matchPackageNames([pkgspec])
for rid in self.disablerepo:
my.repos.disableRepo(rid)
self._enablerepos_with_error_checking(my)
e, m, _ = my.pkgSack.matchPackageNames([pkgspec])
pkgs = e + m pkgs = e + m
if not pkgs: if not pkgs:
pkgs.extend(my.returnPackagesByDep(pkgspec)) pkgs.extend(self.yum_base.returnPackagesByDep(pkgspec))
except Exception as e: except Exception as e:
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e)) self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
@ -613,16 +620,12 @@ class YumModule(YumDnf):
updates = [] updates = []
try: try:
my = self.yum_base() pkgs = self.yum_base.returnPackagesByDep(pkgspec) + \
for rid in self.disablerepo: self.yum_base.returnInstalledPackagesByDep(pkgspec)
my.repos.disableRepo(rid)
self._enablerepos_with_error_checking(my)
pkgs = my.returnPackagesByDep(pkgspec) + my.returnInstalledPackagesByDep(pkgspec)
if not pkgs: if not pkgs:
e, m, _ = my.pkgSack.matchPackageNames([pkgspec]) e, m, _ = self.yum_base.pkgSack.matchPackageNames([pkgspec])
pkgs = e + m pkgs = e + m
updates = my.doPackageLists(pkgnarrow='updates').updates updates = self.yum_base.doPackageLists(pkgnarrow='updates').updates
except Exception as e: except Exception as e:
self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e)) self.module.fail_json(msg="Failure talking to yum: %s" % to_native(e))
@ -653,13 +656,9 @@ class YumModule(YumDnf):
pkgs = [] pkgs = []
try: try:
my = self.yum_base()
for rid in self.disablerepo:
my.repos.disableRepo(rid)
self._enablerepos_with_error_checking(my)
try: try:
pkgs = my.returnPackagesByDep(req_spec) + my.returnInstalledPackagesByDep(req_spec) pkgs = self.yum_base.returnPackagesByDep(req_spec) + \
self.yum_base.returnInstalledPackagesByDep(req_spec)
except Exception as e: except Exception as e:
# If a repo with `repo_gpgcheck=1` is added and the repo GPG # If a repo with `repo_gpgcheck=1` is added and the repo GPG
# key was never accepted, querying this repo will throw an # key was never accepted, querying this repo will throw an
@ -668,14 +667,15 @@ class YumModule(YumDnf):
# the key and try again. # the key and try again.
if 'repomd.xml signature could not be verified' in to_native(e): if 'repomd.xml signature could not be verified' in to_native(e):
self.module.run_command(self.yum_basecmd + ['makecache']) self.module.run_command(self.yum_basecmd + ['makecache'])
pkgs = my.returnPackagesByDep(req_spec) + my.returnInstalledPackagesByDep(req_spec) pkgs = self.yum_base.returnPackagesByDep(req_spec) + \
self.yum_base.returnInstalledPackagesByDep(req_spec)
else: else:
raise raise
if not pkgs: if not pkgs:
e, m, _ = my.pkgSack.matchPackageNames([req_spec]) e, m, _ = self.yum_base.pkgSack.matchPackageNames([req_spec])
pkgs.extend(e) pkgs.extend(e)
pkgs.extend(m) pkgs.extend(m)
e, m, _ = my.rpmdb.matchPackageNames([req_spec]) e, m, _ = self.yum_base.rpmdb.matchPackageNames([req_spec])
pkgs.extend(e) pkgs.extend(e)
pkgs.extend(m) pkgs.extend(m)
except Exception as e: except Exception as e:
@ -767,21 +767,20 @@ class YumModule(YumDnf):
@contextmanager @contextmanager
def set_env_proxy(self): def set_env_proxy(self):
# setting system proxy environment and saving old, if exists # setting system proxy environment and saving old, if exists
my = self.yum_base()
namepass = "" namepass = ""
scheme = ["http", "https"] scheme = ["http", "https"]
old_proxy_env = [os.getenv("http_proxy"), os.getenv("https_proxy")] old_proxy_env = [os.getenv("http_proxy"), os.getenv("https_proxy")]
try: try:
# "_none_" is a special value to disable proxy in yum.conf/*.repo # "_none_" is a special value to disable proxy in yum.conf/*.repo
if my.conf.proxy and my.conf.proxy not in ("_none_",): if self.yum_base.conf.proxy and self.yum_base.conf.proxy not in ("_none_",):
if my.conf.proxy_username: if self.yum_base.conf.proxy_username:
namepass = namepass + my.conf.proxy_username namepass = namepass + self.yum_base.conf.proxy_username
proxy_url = my.conf.proxy proxy_url = self.yum_base.conf.proxy
if my.conf.proxy_password: if self.yum_base.conf.proxy_password:
namepass = namepass + ":" + my.conf.proxy_password namepass = namepass + ":" + self.yum_base.conf.proxy_password
elif '@' in my.conf.proxy: elif '@' in self.yum_base.conf.proxy:
namepass = my.conf.proxy.split('@')[0].split('//')[-1] namepass = self.yum_base.conf.proxy.split('@')[0].split('//')[-1]
proxy_url = my.conf.proxy.replace("{0}@".format(namepass), "") proxy_url = self.yum_base.conf.proxy.replace("{0}@".format(namepass), "")
if namepass: if namepass:
namepass = namepass + '@' namepass = namepass + '@'
@ -792,7 +791,7 @@ class YumModule(YumDnf):
) )
else: else:
for item in scheme: for item in scheme:
os.environ[item + "_proxy"] = my.conf.proxy os.environ[item + "_proxy"] = self.yum_base.conf.proxy
yield yield
except yum.Errors.YumBaseError: except yum.Errors.YumBaseError:
raise raise
@ -1139,6 +1138,7 @@ class YumModule(YumDnf):
# of the process # of the process
# at this point we check to see if the pkg is no longer present # at this point we check to see if the pkg is no longer present
self._yum_base = None # previous YumBase package index is now invalid
for pkg in pkgs: for pkg in pkgs:
if pkg.startswith('@'): if pkg.startswith('@'):
installed = self.is_group_env_installed(pkg) installed = self.is_group_env_installed(pkg)
@ -1471,9 +1471,9 @@ class YumModule(YumDnf):
can be done for remove and absent action can be done for remove and absent action
As solution I would advice to cal As solution I would advice to cal
try: my.repos.disableRepo(disablerepo) try: self.yum_base.repos.disableRepo(disablerepo)
and and
try: my.repos.enableRepo(enablerepo) try: self.yum_base.repos.enableRepo(enablerepo)
right before any yum_cmd is actually called regardless right before any yum_cmd is actually called regardless
of yum action. of yum action.
@ -1490,19 +1490,14 @@ class YumModule(YumDnf):
if self.update_cache: if self.update_cache:
self.module.run_command(self.yum_basecmd + ['clean', 'expire-cache']) self.module.run_command(self.yum_basecmd + ['clean', 'expire-cache'])
my = self.yum_base()
try: try:
if self.disablerepo: current_repos = self.yum_base.repos.repos.keys()
for rid in self.disablerepo:
my.repos.disableRepo(rid)
current_repos = my.repos.repos.keys()
if self.enablerepo: if self.enablerepo:
try: try:
self._enablerepos_with_error_checking(my) new_repos = self.yum_base.repos.repos.keys()
new_repos = my.repos.repos.keys()
for i in new_repos: for i in new_repos:
if i not in current_repos: if i not in current_repos:
rid = my.repos.getRepo(i) rid = self.yum_base.repos.getRepo(i)
a = rid.repoXML.repoid # nopep8 - https://github.com/ansible/ansible/pull/21475#pullrequestreview-22404868 a = rid.repoXML.repoid # nopep8 - https://github.com/ansible/ansible/pull/21475#pullrequestreview-22404868
current_repos = new_repos current_repos = new_repos
except yum.Errors.YumBaseError as e: except yum.Errors.YumBaseError as e:
@ -1598,13 +1593,9 @@ class YumModule(YumDnf):
# the system then users will see an error message using the yum API. # the system then users will see an error message using the yum API.
# Use repoquery in those cases. # Use repoquery in those cases.
my = self.yum_base()
# A sideeffect of accessing conf is that the configuration is
# loaded and plugins are discovered
my.conf
repoquery = None repoquery = None
try: try:
yum_plugins = my.plugins._plugins yum_plugins = self.yum_base.plugins._plugins
except AttributeError: except AttributeError:
pass pass
else: else:

Write Preview
Loading…
Cancel
Save