[2.7] docsite: remove lexers which have been fixed in Pygments 2.4.0 (#58101)

* docsite: remove lexers which have been fixed in Pygments 2.4.0 (#57508)

* Remove lexers which have been fixed in Pygments 2.4.0.
* Add Pygments >= 2.4.0 to test runner.
* Fix pages that triggered lexer errors.

Co-Authored-By: Sviatoslav Sydorenko <wk.cvs.github@sydorenko.org.ua>
(cherry picked from commit 505c99265c)

* fixes 'could not lex literal_block' errors
pull/58439/head
Felix Fontein 7 years ago committed by Alicia Cozine
parent e5595cddfa
commit b7f15c69b1

@ -36,447 +36,11 @@
from __future__ import absolute_import, print_function
from pygments.lexer import LexerContext, ExtendedRegexLexer, DelegatingLexer, RegexLexer, bygroups, include
from pygments.lexers import DjangoLexer, DiffLexer
from pygments.lexer import DelegatingLexer, RegexLexer, bygroups, include
from pygments.lexers import DiffLexer
from pygments import token
class AnsibleYamlLexerContext(LexerContext):
"""Indentation context for the YAML lexer."""
def __init__(self, *args, **kwds):
super(AnsibleYamlLexerContext, self).__init__(*args, **kwds)
self.indent_stack = []
self.indent = -1
self.next_indent = 0
self.block_scalar_indent = None
class AnsibleYamlLexer(ExtendedRegexLexer):
"""
Lexer for `YAML <http://yaml.org/>`_, a human-friendly data serialization
language.
.. versionadded:: 0.11
"""
name = 'YAML'
aliases = ['yaml']
filenames = ['*.yaml', '*.yml']
mimetypes = ['text/x-yaml']
def something(token_class):
"""Do not produce empty tokens."""
def callback(lexer, match, context):
text = match.group()
if not text:
return
yield match.start(), token_class, text
context.pos = match.end()
return callback
def reset_indent(token_class):
"""Reset the indentation levels."""
def callback(lexer, match, context):
text = match.group()
context.indent_stack = []
context.indent = -1
context.next_indent = 0
context.block_scalar_indent = None
yield match.start(), token_class, text
context.pos = match.end()
return callback
def save_indent(token_class, start=False):
"""Save a possible indentation level."""
def callback(lexer, match, context):
text = match.group()
extra = ''
if start:
context.next_indent = len(text)
if context.next_indent < context.indent:
while context.next_indent < context.indent:
context.indent = context.indent_stack.pop()
if context.next_indent > context.indent:
extra = text[context.indent:]
text = text[:context.indent]
else:
context.next_indent += len(text)
if text:
yield match.start(), token_class, text
if extra:
yield match.start() + len(text), token_class.Error, extra
context.pos = match.end()
return callback
def set_indent(token_class, implicit=False):
"""Set the previously saved indentation level."""
def callback(lexer, match, context):
text = match.group()
if context.indent < context.next_indent:
context.indent_stack.append(context.indent)
context.indent = context.next_indent
if not implicit:
context.next_indent += len(text)
yield match.start(), token_class, text
context.pos = match.end()
return callback
def set_block_scalar_indent(token_class):
"""Set an explicit indentation level for a block scalar."""
def callback(lexer, match, context):
text = match.group()
context.block_scalar_indent = None
if not text:
return
increment = match.group(1)
if increment:
current_indent = max(context.indent, 0)
increment = int(increment)
context.block_scalar_indent = current_indent + increment
if text:
yield match.start(), token_class, text
context.pos = match.end()
return callback
def parse_block_scalar_empty_line(indent_token_class, content_token_class):
"""Process an empty line in a block scalar."""
def callback(lexer, match, context):
text = match.group()
if (context.block_scalar_indent is None or
len(text) <= context.block_scalar_indent):
if text:
yield match.start(), indent_token_class, text
else:
indentation = text[:context.block_scalar_indent]
content = text[context.block_scalar_indent:]
yield match.start(), indent_token_class, indentation
yield (match.start() + context.block_scalar_indent,
content_token_class, content)
context.pos = match.end()
return callback
def parse_block_scalar_indent(token_class):
"""Process indentation spaces in a block scalar."""
def callback(lexer, match, context):
text = match.group()
if context.block_scalar_indent is None:
if len(text) <= max(context.indent, 0):
context.stack.pop()
context.stack.pop()
return
context.block_scalar_indent = len(text)
else:
if len(text) < context.block_scalar_indent:
context.stack.pop()
context.stack.pop()
return
if text:
yield match.start(), token_class, text
context.pos = match.end()
return callback
def parse_plain_scalar_indent(token_class):
"""Process indentation spaces in a plain scalar."""
def callback(lexer, match, context):
text = match.group()
if len(text) <= context.indent:
context.stack.pop()
context.stack.pop()
return
if text:
yield match.start(), token_class, text
context.pos = match.end()
return callback
tokens = {
# the root rules
'root': [
# ignored whitespaces
(r'[ ]+(?=#|$)', token.Text),
# line breaks
(r'\n+', token.Text),
# a comment
(r'#[^\n]*', token.Comment.Single),
# the '%YAML' directive
(r'^%YAML(?=[ ]|$)', reset_indent(token.Name.Tag), 'yaml-directive'),
# the %TAG directive
(r'^%TAG(?=[ ]|$)', reset_indent(token.Name.Tag), 'tag-directive'),
# document start and document end indicators
(r'^(?:---|\.\.\.)(?=[ ]|$)', reset_indent(token.Name.Namespace),
'block-line'),
# indentation spaces
(r'[ ]*(?!\s|$)', save_indent(token.Text, start=True),
('block-line', 'indentation')),
],
# trailing whitespaces after directives or a block scalar indicator
'ignored-line': [
# ignored whitespaces
(r'[ ]+(?=#|$)', token.Text),
# a comment
(r'#[^\n]*', token.Comment.Single),
# line break
(r'\n', token.Text, '#pop:2'),
],
# the %YAML directive
'yaml-directive': [
# the version number
(r'([ ]+)([0-9]+\.[0-9]+)',
bygroups(token.Text, token.Number), 'ignored-line'),
],
# the %YAG directive
'tag-directive': [
# a tag handle and the corresponding prefix
(r'([ ]+)(!|![\w-]*!)'
r'([ ]+)(!|!?[\w;/?:@&=+$,.!~*\'()\[\]%-]+)',
bygroups(token.Text, token.Keyword.Type, token.Text, token.Keyword.Type),
'ignored-line'),
],
# block scalar indicators and indentation spaces
'indentation': [
# trailing whitespaces are ignored
(r'[ ]*$', something(token.Text), '#pop:2'),
# whitespaces preceeding block collection indicators
(r'[ ]+(?=[?:-](?:[ ]|$))', save_indent(token.Text)),
# block collection indicators
(r'[?:-](?=[ ]|$)', set_indent(token.Punctuation.Indicator)),
# the beginning a block line
(r'[ ]*', save_indent(token.Text), '#pop'),
],
# an indented line in the block context
'block-line': [
# the line end
(r'[ ]*(?=#|$)', something(token.Text), '#pop'),
# whitespaces separating tokens
(r'[ ]+', token.Text),
# key with colon
(r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)',
bygroups(token.Name.Tag, set_indent(token.Punctuation, implicit=True))),
# tags, anchors and aliases,
include('descriptors'),
# block collections and scalars
include('block-nodes'),
# flow collections and quoted scalars
include('flow-nodes'),
# a plain scalar
(r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`-]|[?:-]\S)',
something(token.Name.Variable),
'plain-scalar-in-block-context'),
],
# tags, anchors, aliases
'descriptors': [
# a full-form tag
(r'!<[\w#;/?:@&=+$,.!~*\'()\[\]%-]+>', token.Keyword.Type),
# a tag in the form '!', '!suffix' or '!handle!suffix'
(r'!(?:[\w-]+!)?'
r'[\w#;/?:@&=+$,.!~*\'()\[\]%-]+', token.Keyword.Type),
# an anchor
(r'&[\w-]+', token.Name.Label),
# an alias
(r'\*[\w-]+', token.Name.Variable),
],
# block collections and scalars
'block-nodes': [
# implicit key
(r':(?=[ ]|$)', set_indent(token.Punctuation.Indicator, implicit=True)),
# literal and folded scalars
(r'[|>]', token.Punctuation.Indicator,
('block-scalar-content', 'block-scalar-header')),
],
# flow collections and quoted scalars
'flow-nodes': [
# a flow sequence
(r'\[', token.Punctuation.Indicator, 'flow-sequence'),
# a flow mapping
(r'\{', token.Punctuation.Indicator, 'flow-mapping'),
# a single-quoted scalar
(r'\'', token.String, 'single-quoted-scalar'),
# a double-quoted scalar
(r'\"', token.String, 'double-quoted-scalar'),
],
# the content of a flow collection
'flow-collection': [
# whitespaces
(r'[ ]+', token.Text),
# line breaks
(r'\n+', token.Text),
# a comment
(r'#[^\n]*', token.Comment.Single),
# simple indicators
(r'[?:,]', token.Punctuation.Indicator),
# tags, anchors and aliases
include('descriptors'),
# nested collections and quoted scalars
include('flow-nodes'),
# a plain scalar
(r'(?=[^\s?:,\[\]{}#&*!|>\'"%@`])',
something(token.Name.Variable),
'plain-scalar-in-flow-context'),
],
# a flow sequence indicated by '[' and ']'
'flow-sequence': [
# include flow collection rules
include('flow-collection'),
# the closing indicator
(r'\]', token.Punctuation.Indicator, '#pop'),
],
# a flow mapping indicated by '{' and '}'
'flow-mapping': [
# key with colon
(r'([^,:?\[\]{}\n]+)(:)(?=[ ]|$)',
bygroups(token.Name.Tag, token.Punctuation)),
# include flow collection rules
include('flow-collection'),
# the closing indicator
(r'\}', token.Punctuation.Indicator, '#pop'),
],
# block scalar lines
'block-scalar-content': [
# line break
(r'\n', token.Text),
# empty line
(r'^[ ]+$',
parse_block_scalar_empty_line(token.Text, token.Name.Constant)),
# indentation spaces (we may leave the state here)
(r'^[ ]*', parse_block_scalar_indent(token.Text)),
# line content
(r'[\S\t ]+', token.Name.Constant),
],
# the content of a literal or folded scalar
'block-scalar-header': [
# indentation indicator followed by chomping flag
(r'([1-9])?[+-]?(?=[ ]|$)',
set_block_scalar_indent(token.Punctuation.Indicator),
'ignored-line'),
# chomping flag followed by indentation indicator
(r'[+-]?([1-9])?(?=[ ]|$)',
set_block_scalar_indent(token.Punctuation.Indicator),
'ignored-line'),
],
# ignored and regular whitespaces in quoted scalars
'quoted-scalar-whitespaces': [
# leading and trailing whitespaces are ignored
(r'^[ ]+', token.Text),
(r'[ ]+$', token.Text),
# line breaks are ignored
(r'\n+', token.Text),
# other whitespaces are a part of the value
(r'[ ]+', token.Name.Variable),
],
# single-quoted scalars
'single-quoted-scalar': [
# include whitespace and line break rules
include('quoted-scalar-whitespaces'),
# escaping of the quote character
(r'\'\'', token.String.Escape),
# regular non-whitespace characters
(r'[^\s\']+', token.String),
# the closing quote
(r'\'', token.String, '#pop'),
],
# double-quoted scalars
'double-quoted-scalar': [
# include whitespace and line break rules
include('quoted-scalar-whitespaces'),
# escaping of special characters
(r'\\[0abt\tn\nvfre "\\N_LP]', token.String),
# escape codes
(r'\\(?:x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})',
token.String.Escape),
# regular non-whitespace characters
(r'[^\s"\\]+', token.String),
# the closing quote
(r'"', token.String, '#pop'),
],
# the beginning of a new line while scanning a plain scalar
'plain-scalar-in-block-context-new-line': [
# empty lines
(r'^[ ]+$', token.Text),
# line breaks
(r'\n+', token.Text),
# document start and document end indicators
(r'^(?=---|\.\.\.)', something(token.Name.Namespace), '#pop:3'),
# indentation spaces (we may leave the block line state here)
(r'^[ ]*', parse_plain_scalar_indent(token.Text), '#pop'),
],
# a plain scalar in the block context
'plain-scalar-in-block-context': [
# the scalar ends with the ':' indicator
(r'[ ]*(?=:[ ]|:$)', something(token.Text), '#pop'),
# the scalar ends with whitespaces followed by a comment
(r'[ ]+(?=#)', token.Text, '#pop'),
# trailing whitespaces are ignored
(r'[ ]+$', token.Text),
# line breaks are ignored
(r'\n+', token.Text, 'plain-scalar-in-block-context-new-line'),
# other whitespaces are a part of the value
(r'[ ]+', token.Literal.Scalar.Plain),
# regular non-whitespace characters
(r'(?::(?!\s)|[^\s:])+', token.Literal.Scalar.Plain),
],
# a plain scalar is the flow context
'plain-scalar-in-flow-context': [
# the scalar ends with an indicator character
(r'[ ]*(?=[,:?\[\]{}])', something(token.Text), '#pop'),
# the scalar ends with a comment
(r'[ ]+(?=#)', token.Text, '#pop'),
# leading and trailing whitespaces are ignored
(r'^[ ]+', token.Text),
(r'[ ]+$', token.Text),
# line breaks are ignored
(r'\n+', token.Text),
# other whitespaces are a part of the value
(r'[ ]+', token.Name.Variable),
# regular non-whitespace characters
(r'[^\s,:?\[\]{}]+', token.Name.Variable),
],
}
def get_tokens_unprocessed(self, text=None, context=None):
if context is None:
context = AnsibleYamlLexerContext(text, 0)
return super(AnsibleYamlLexer, self).get_tokens_unprocessed(text, context)
class AnsibleYamlJinjaLexer(DelegatingLexer):
"""
Subclass of the `DjangoLexer` that highlights unlexed data with the
`AnsibleYamlLexer`.
Commonly used in Saltstack salt states.
.. versionadded:: 2.0
"""
name = 'YAML+Jinja'
aliases = ['yaml+jinja']
filenames = ['*.sls']
mimetypes = ['text/x-yaml+jinja']
def __init__(self, **options):
super(AnsibleYamlJinjaLexer, self).__init__(AnsibleYamlLexer, DjangoLexer, **options)
class AnsibleOutputPrimaryLexer(RegexLexer):
name = 'Ansible-output-primary'
@ -609,7 +173,7 @@ def setup(app):
""" Initializer for Sphinx extension API.
See http://www.sphinx-doc.org/en/stable/extdev/index.html#dev-extensions.
"""
for lexer in [AnsibleYamlLexer(startinline=True), AnsibleYamlJinjaLexer(startinline=True), AnsibleOutputLexer(startinline=True)]:
for lexer in [AnsibleOutputLexer(startinline=True)]:
app.add_lexer(lexer.name, lexer)
for alias in lexer.aliases:
app.add_lexer(alias, lexer)

@ -56,7 +56,7 @@ To create a new module:
3. Paste the content below into your new module file. It includes the :ref:`required Ansible format and documentation <developing_modules_documenting>` and some example code.
4. Modify and extend the code to do what you want your new module to do. See the :ref:`programming tips <developing_modules_best_practices>` and :ref:`Python 3 compatibility <developing_python_3>` pages for pointers on writing clean, concise module code.
.. code:: python
.. code-block:: python
#!/usr/bin/python

@ -53,7 +53,9 @@ Or for the openstack plugin:
# clouds.yml
plugin: openstack
The ``auto`` inventory plugin is enabled by default and works by using the ``plugin`` field to indicate the plugin that should attempt to parse it. You can configure the whitelist/precedence of inventory plugins used to parse source using the `ansible.cfg` ['inventory'] ``enable_plugins`` list. After enabling the plugin and providing any required options you can view the populated inventory with ``ansible-inventory -i demo.aws_ec2.yml --graph``::
The ``auto`` inventory plugin is enabled by default and works by using the ``plugin`` field to indicate the plugin that should attempt to parse it. You can configure the whitelist/precedence of inventory plugins used to parse source using the `ansible.cfg` ['inventory'] ``enable_plugins`` list. After enabling the plugin and providing any required options you can view the populated inventory with ``ansible-inventory -i demo.aws_ec2.yml --graph``:
.. code-block:: text
@all:
|--@aws_ec2:
@ -86,7 +88,9 @@ You can create dynamic groups using host variables with the constructed ``keyed_
# set the ansible_host variable to connect with the private IP address without changing the hostname
ansible_host: private_ip_address
Now the output of ``ansible-inventory -i demo.aws_ec2.yml --graph``::
Now the output of ``ansible-inventory -i demo.aws_ec2.yml --graph``:
.. code-block:: text
@all:
|--@aws_ec2:
@ -109,7 +113,7 @@ If a host does not have the variables in the configuration above (i.e. ``tags.Na
Plugin List
-----------
You can use ``ansible-doc -t inventory -l`` to see the list of available plugins.
You can use ``ansible-doc -t inventory -l`` to see the list of available plugins.
Use ``ansible-doc -t inventory <plugin name>`` to see plugin-specific documentation and examples.
.. toctree:: :maxdepth: 1

@ -39,24 +39,20 @@ Using Lookup Plugins
Lookup plugins can be used anywhere you can use templating in Ansible: in a play, in variables file, or in a Jinja2 template for the :ref:`template <template_module>` module.
.. code-block:: yaml
.. code-block:: YAML+Jinja
vars:
file_contents: "{{lookup('file', 'path/to/file.txt')}}"
Lookups are an integral part of loops. Wherever you see ``with_``, the part after the underscore is the name of a lookup.
This is also the reason most lookups output lists and take lists as input; for example, ``with_items`` uses the :doc:`items <lookup/items>` lookup:
.. code-block:: yaml
This is also the reason most lookups output lists and take lists as input; for example, ``with_items`` uses the :doc:`items <lookup/items>` lookup::
tasks:
- name: count to 3
debug: msg={{item}}
with_items: [1, 2, 3]
You can combine lookups with :ref:`playbooks_filters`, :ref:`playbooks_tests` and even each other to do some complex data generation and manipulation. For example:
.. code-block:: yaml
You can combine lookups with :ref:`playbooks_filters`, :ref:`playbooks_tests` and even each other to do some complex data generation and manipulation. For example::
tasks:
- name: valid but useless and over complicated chained lookups and filters
@ -75,6 +71,8 @@ To ignore errors::
- name: file doesnt exist, but i dont care .. file plugin itself warns anyways ...
debug: msg="{{ lookup('file', '/idontexist', errors='ignore') }}"
.. code-block:: ansible-output
[WARNING]: Unable to find '/idontexist' in expected paths (use -vvvvv to see paths)
ok: [localhost] => {
@ -87,6 +85,8 @@ To get a warning instead of a failure::
- name: file doesnt exist, let me know, but continue
debug: msg="{{ lookup('file', '/idontexist', errors='warn') }}"
.. code-block:: ansible-output
[WARNING]: Unable to find '/idontexist' in expected paths (use -vvvvv to see paths)
[WARNING]: An unhandled exception occurred while running the lookup plugin 'file'. Error was a <class 'ansible.errors.AnsibleError'>, original message: could not locate file in lookup: /idontexist
@ -101,6 +101,8 @@ Fatal error (the default)::
- name: file doesnt exist, FAIL (this is the default)
debug: msg="{{ lookup('file', '/idontexist', errors='strict') }}"
.. code-block:: ansible-output
[WARNING]: Unable to find '/idontexist' in expected paths (use -vvvvv to see paths)
fatal: [localhost]: FAILED! => {"msg": "An unhandled exception occurred while running the lookup plugin 'file'. Error was a <class 'ansible.errors.AnsibleError'>, original message: could not locate file in lookup: /idontexist"}
@ -118,7 +120,9 @@ The default behavior of ``lookup`` is to return a string of comma separated valu
This was done primarily to provide an easier and more consistent interface for interacting with the new ``loop`` keyword, while maintaining backwards compatibiltiy with other uses of ``lookup``.
The following examples are equivalent::
The following examples are equivalent:
.. code-block:: jinja
lookup('dict', dict_variable, wantlist=True)
@ -126,7 +130,9 @@ The following examples are equivalent::
As demonstrated above the behavior of ``wantlist=True`` is implicit when using ``query``.
Additionally, ``q`` was introduced as a shortform of ``query``::
Additionally, ``q`` was introduced as a shortform of ``query``:
.. code-block:: jinja
q('dict', dict_variable)

@ -5,7 +5,7 @@ YAML Syntax
===========
This page provides a basic overview of correct YAML syntax, which is how Ansible
playbooks (our configuration management language) are expressed.
playbooks (our configuration management language) are expressed.
We use YAML because it is easier for humans to read and write than other common
data formats like XML or JSON. Further, there are libraries available in most
@ -18,7 +18,7 @@ is used in practice.
YAML Basics
-----------
For Ansible, nearly every YAML file starts with a list.
For Ansible, nearly every YAML file starts with a list.
Each item in the list is a list of key/value pairs, commonly
called a "hash" or a "dictionary". So, we need to know how
to write lists and dictionaries in YAML.
@ -153,7 +153,7 @@ Because of this, the following is going to result in a YAML syntax error::
You will want to quote hash values using colons followed by a space or the end of the line::
foo: 'somebody said I should put a colon here: so I did'
windows_drive: 'c:'
...and then the colon will be preserved.
@ -161,7 +161,7 @@ You will want to quote hash values using colons followed by a space or the end o
Alternatively, you can use double quotes::
foo: "somebody said I should put a colon here: so I did"
windows_drive: "c:"
The difference between single quotes and double quotes is that in double quotes
@ -171,7 +171,9 @@ you can use escapes::
The list of allowed escapes can be found in the YAML Specification under "Escape Sequences" (YAML 1.1) or "Escape Characters" (YAML 1.2).
The following is invalid YAML::
The following is invalid YAML:
.. code-block:: text
foo: "an escaped \' single quote"
@ -237,4 +239,3 @@ value::
implementing
`YAML 1.2 Specification <http://yaml.org/spec/1.2/spec.html>`_
For completeness, YAML 1.2 is the successor of 1.1

@ -388,24 +388,29 @@ The YAML specification considers the following `escape sequences <http://yaml.or
Here are some examples on how to write Windows paths::
GOOD
# GOOD
tempdir: C:\Windows\Temp
WORKS
# WORKS
tempdir: 'C:\Windows\Temp'
tempdir: "C:\\Windows\\Temp"
BAD, BUT SOMETIMES WORKS
# BAD, BUT SOMETIMES WORKS
tempdir: C:\\Windows\\Temp
tempdir: 'C:\\Windows\\Temp'
tempdir: C:/Windows/Temp
FAILS
This is an example which will fail:
.. code-block:: text
# FAILS
tempdir: "C:\Windows\Temp"
This example shows the use of single quotes when they are required::
---
# example of single quotes when they are required
- name: copy tomcat config
- name: Copy tomcat config
win_copy:
src: log4j.xml
dest: '{{tc_home}}\lib\log4j.xml'

@ -8,6 +8,7 @@ pylint == 1.7.4 ; python_version >= '3.5' # versions before 1.7.1 hang or fail t
pylint == 1.6.5 ; python_version <= '2.7' # versions after 1.6.5 hang or fail during test on python 2.x
sphinx < 1.6 ; python_version < '2.7' # sphinx 1.6 and later require python 2.7 or later
sphinx < 1.8 ; python_version >= '2.7' # sphinx 1.8 and later are currently incompatible with rstcheck 3.3
pygments >= 2.4.0 # Pygments 2.4.0 includes bugfixes for YAML and YAML+Jinja lexers
wheel < 0.30.0 ; python_version < '2.7' # wheel 0.30.0 and later require python 2.7 or later
yamllint != 1.8.0 ; python_version < '2.7' # yamllint 1.8.0 requires python 2.7+ while earlier/later versions do not
isort < 4.2.8 # 4.2.8 changes import sort order requirements which breaks previously passing pylint tests

Loading…
Cancel
Save