diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..1f68a45b --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,19 @@ + +API Reference +============= + + +econtext.core +############# + +.. automodule:: econtext.core + :members: + :undoc-members: + + +econtext.master +############### + +.. automodule:: econtext.master + :members: + :undoc-members: diff --git a/docs/conf.py b/docs/conf.py index 907adda2..8da56957 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,285 +1,22 @@ -# -*- coding: utf-8 -*- -# -# econtext documentation build configuration file, created by -# sphinx-quickstart on Thu Aug 11 18:16:36 2016. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - import sys import os -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'econtext' -copyright = u'2016, David Wilson' author = u'David Wilson' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = u'master' -# The full version, including alpha/beta/rc tags. -release = u'master' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. +copyright = u'2016, David Wilson' exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'alabaster' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". +extensions = ['sphinx.ext.autodoc'] +html_show_sourcelink = False +html_show_sphinx = False +html_sidebars = {'**': ['globaltoc.html']} html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. +html_theme = 'alabaster' htmlhelp_basename = 'econtextdoc' - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'econtext.tex', u'econtext Documentation', - u'David Wilson', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'econtext', u'econtext Documentation', - [author], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - (master_doc, 'econtext', u'econtext Documentation', - author, 'econtext', 'One line description of project.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +language = None +master_doc = 'index' +project = u'econtext' +pygments_style = 'sphinx' +release = u'master' +source_suffix = '.rst' +templates_path = ['_templates'] +todo_include_todos = False +version = u'master' diff --git a/docs/getting_started.rst b/docs/getting_started.rst new file mode 100644 index 00000000..97258614 --- /dev/null +++ b/docs/getting_started.rst @@ -0,0 +1,5 @@ + +Getting Started +=============== + +xxx diff --git a/docs/howitworks.rst b/docs/howitworks.rst new file mode 100644 index 00000000..56584d3e --- /dev/null +++ b/docs/howitworks.rst @@ -0,0 +1,7 @@ + +How econtext Works +------------------ + +blah blah + + diff --git a/docs/index.rst b/docs/index.rst index ff41eb34..83a0f941 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,22 +1,209 @@ -.. econtext documentation master file, created by - sphinx-quickstart on Thu Aug 11 18:16:36 2016. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. -Welcome to econtext's documentation! -==================================== +Python Execution Contexts +========================= -Contents: +**4.98KiB of sugar and no fat!** .. toctree:: - :maxdepth: 2 + :maxdepth: 1 + self + howitworks + getting_started + api -Indices and tables -================== +Introduction +------------ -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` +The Python ``econtext`` package implements external *execution contexts*: an +execution context is somewhere you can run Python code external to your main +process, even on a remote machine. +**There is no requirement for installing packages, copying files around, +writing shell scripts, upfront configuration, or providing any secondary link +to the remote machine**. Due to the origins of econtext for use in managing +potentially damaged infrastructure, the remote machine need not even have a +writeable filesystem. + +It is not intended as a generic RPC framework, the goal is to provide a robust +and efficient low-level API on which tools like **Salt** or **Ansible** can be +built, and while the API is quite friendly and similar in scope to **Fabric**, +ultimately it should not be used directly by consumer software. + +The primary focus is to centralize and perfect the intricate dance required to +run Python code safely and efficiently on a remote machine, while avoiding +temporary files or large chunks of error-prone shell scripts. + + +Automatic Bootstrap +################### + +The package's main feature is enabling your Python program to bootstrap and +communicate with new Python programs under its control running on remote +machines, **using only an existing installed Python interpreter and SSH +client**, something that by default can be found on almost all contemporary +machines in the wild. To accomplish bootstrap, econtext uses a single 500 byte +SSH command line and 5KB of data sent to stdin of the remote SSH connection. + +.. code:: + + $ python preamble_size.py + SSH command size: 411 + Preamble size: 5098 (4.98KiB) + econtext.master size: 2403 (2.35KiB) + +Once bootstrapped, the remote process is configured with a customizable +**argv[0]**, readily visible to system administrators of the remote machine +using the UNIX **ps** command: + +.. code:: + + 20051 ? Ss 0:00 \_ sshd: dmw [priv] + 20053 ? S 0:00 | \_ sshd: dmw@notty + 20054 ? Ssl 0:00 | \_ econtext:dmw@Eldil.home:22476 + 20103 ? S 0:00 | \_ tar zxvf myapp.tar.gz + +The example context was started by UID ``dmw`` on host ``Eldil.home``, process +ID ``22476``. + + +IO Multiplexer +############## + +The bootstrap includes a compact IO multiplexer (like Twisted or asyncio) that +allows it to perform work in the background while executing your program's +code. For example, the remote context can be used to connect to a new user on +the remote machine using **sudo**, or as an intermediary for extending the +program's domain of control outward to other machines, enabling your program to +manipulate machines behind a firewall, or to allow its data plane to match your +network topology. + +The multiplexer also ensures the remote process is terminated if your Python +program crashes, communication is lost, or the application code running in the +context has hung. + + +Module Forwarder +################ + +In addition to an IO multiplexer, the external context is configured with a +custom **PEP-302 importer** that forwards requests for unknown Python modules +back to the host machine. When your program asks an external context to execute +code from an unknown module, all requisite modules are transferred +automatically and imported entirely in RAM without need for further +configuration. + +.. code-block:: python + + import myapp.mypkg.mymodule + + # myapp/__init__.py, myapp/mypkg/__init__.py, and myapp/mypkg/mymodule.py + # are transferred automatically. + print context.call(myapp.mymodule.my_function) + + +Logging Forwarder +################# + +The 5KB bootstrap configures the remote process's Python logging package to +forward all logs back to the local process, enabling management of program logs +in one location. + +.. code:: + + 18:15:29 D econtext.ctx.k3: econtext: SlaveModuleImporter.find_module('econtext.zlib') + 18:15:29 D econtext.ctx.k3: econtext: _dispatch_calls((1002L, False, 'posix', None, 'system', ('ls -l /proc/self/fd',), {})) + + +Stdio Forwarder +############### + +To ease porting of crusty old infrastructure code to pure Python, the bootstrap +redirects stdio for itself and any child processes back into the logging +framework. This allows use of functions as basic as **os.system('hostname; +uptime')** without further need to capture or manage output. + +.. code:: + + 18:17:28 D econtext.ctx.k3: econtext: _dispatch_calls((1002L, False, 'posix', None, 'system', ('hostname; uptime',), {})) + 18:17:56 I econtext.ctx.k3: stdout: k3 + 18:17:56 I econtext.ctx.k3: stdout: 17:37:10 up 562 days, 2:25, 5 users, load average: 1.24, 1.13, 1.14 + + +Blocking Code Friendly +###################### + +Within each process, a private thread runs the I/O multiplexer, leaving the +main thread and any additional application threads free to perform useful work. + +While econtext is internally asynchronous, it hides this asynchrony from +consumer code. This is since writing asynchronous code is mostly a foreign +concept to the target application of managing infrastructure. It should be +possible to rewrite a shell script in Python without significant restructuring, +or mind-bending feats of comprehension to understand control flow. + +Before: + +.. code-block:: sh + + #!/bin/bash + # Install our application. + + tar zxvf app.tar.gz + +After: + +.. code-block:: python + + def install_app(): + """ + Install our application. + """ + os.system('tar zxvf app.tar.gz') + + context.call(install_app) + +Exceptions raised by function calls are propagated back to the parent program, +and timeouts can be configured to ensure failed calls do not block progress of +the parent. + + +Event-driven IO +############### + +Code running in a remote context can be connected to a *Channel*. Channels are +used to send data asynchronously back to the parent, without further need for +the parent to poll for changes. This is useful for monitoring systems managing +a large fleet of machines, or to alert the parent of unexpected state changes. + +.. code-block:: python + + def tail_log_file(channel, path='/var/log/messages'): + """ + Forward new lines in a log file to the parent. + """ + size = os.path.getsize(path) + + while channel.open(): + new_size = os.path.getsize(path) + if new_size == size: + time.sleep(1) + continue + elif new_size < size: + size = 0 + + fp = file(path, 'r') + fp.seek(size) + channel.send(fp.read(new_size - size)) + fp.close() + size = new_size + + +Compatibility +############# + +The package is written using syntax compatible all the way back to **Python +2.4** released November 2004, making it suitable for managing a fleet of +potentially ancient corporate hardware. For example econtext can be used out of +the box against Red Hat Enterprise Linux 5, which was released in 2007. diff --git a/econtext/core.py b/econtext/core.py index fe4c4f15..0663e158 100644 --- a/econtext/core.py +++ b/econtext/core.py @@ -1,5 +1,7 @@ """ -Python external execution contexts. +This module implements most package functionality, but remains separate from +non-essential code in order to reduce its size, as it is also implements the +bootstrap code. """ import Queue diff --git a/econtext/master.py b/econtext/master.py index b45d89bc..82616241 100644 --- a/econtext/master.py +++ b/econtext/master.py @@ -1,3 +1,8 @@ +""" +This module implements functionality required by master processes, such as +starting new contexts via SSH. Its size is also restricted, since it must be +sent to any context that will be used to establish additional child contexts. +""" import commands import getpass