Version bump for 0.3 release

Conflicts:
	lib/ansible/playbook.py
	lib/ansible/runner.py
	library/apt

.gitignore

@@ -7,3 +7,6 @@ build
 MANIFEST
 dist
 rpm-build
+# Eclipse/PyDev stuff...
+.project
+.pydevproject

MANIFEST.in

@@ -1,5 +1,6 @@
-include README.md ansible.spec
+include README.md packaging/rpm/ansible.spec
 include examples/hosts
+include packaging/distutils/setup.py
 recursive-include docs *
 recursive-include library *
 include Makefile

Makefile

@@ -1,25 +1,59 @@
 #!/usr/bin/make
 
+########################################################
+# Makefile for Ansible
+#
+# useful targets:
+#   make sdist ---------------- produce a tarball
+#   make rpm  ----------------- produce RPMs
+#   make debian --------------- produce a dpkg (FIXME?)
+#   make docs ----------------- rebuild the manpages (results are checked in)
+#   make tests ---------------- run the tests
+#   make pyflakes, make pep8 -- source code checks  
+
+########################################################
+# variable section
+
 NAME = "ansible"
+
+# Manpages are currently built with asciidoc -- would like to move to markdown
+# This doesn't evaluate until it's called. The -D argument is the
+# directory of the target file ($@), kinda like `dirname`.
 ASCII2MAN = a2x -D $(dir $@) -d manpage -f manpage $<
 ASCII2HTMLMAN = a2x -D docs/html/man/ -d manpage -f xhtml
 MANPAGES := docs/man/man1/ansible.1 docs/man/man1/ansible-playbook.1
+
 SITELIB = $(shell python -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")
-RPMVERSION := $(shell awk '/Version/{print $$2; exit}' < ansible.spec | cut -d "%" -f1)
-RPMRELEASE := $(shell awk '/Release/{print $$2; exit}' < ansible.spec | cut -d "%" -f1)
-RPMNVR = "$(NAME)-$(RPMVERSION)-$(RPMRELEASE)"
+
+# VERSION file provides one place to update the software version
+VERSION := $(shell cat VERSION)
+
+# RPM build parameters
+RPMSPECDIR= packaging/rpm
+RPMSPEC = $(RPMSPECDIR)/ansible.spec
+RPMVERSION := $(shell awk '/Version/{print $$2; exit}' < $(RPMSPEC) | cut -d "%" -f1)
+RPMRELEASE := $(shell awk '/Release/{print $$2; exit}' < $(RPMSPEC) | cut -d "%" -f1)
+RPMDIST = $(shell rpm --eval '%dist')
+RPMNVR = "$(NAME)-$(RPMVERSION)-$(RPMRELEASE)$(RPMDIST)"
+
+########################################################
 
 all: clean python
 
 tests: 
 	PYTHONPATH=./lib nosetests -v
 
+# To force a rebuild of the docs run 'touch VERSION && make docs'
 docs: $(MANPAGES)
 
-%.1: %.1.asciidoc
-	$(ASCII2MAN)
+# Regenerate %.1.asciidoc if %.1.asciidoc.in has been modified more
+# recently than %.1.asciidoc.
+%.1.asciidoc: %.1.asciidoc.in
+	sed "s/%VERSION%/$(VERSION)/" $< > $@
 
-%.5: %.5.asciidoc
+# Regenerate %.1 if %.1.asciidoc or VERSION has been modified more
+# recently than %.1. (Implicitly runs the %.1.asciidoc recipe)
+%.1: %.1.asciidoc VERSION
 	$(ASCII2MAN)
 
 loc:
@@ -29,26 +63,30 @@ pep8:
 	@echo "#############################################"
 	@echo "# Running PEP8 Compliance Tests"
 	@echo "#############################################"
-	pep8 -r --ignore=E501,E221,W291,W391,E302,E251,E203,W293,E231,E303,E201,E225 lib/ bin/
+	pep8 -r --ignore=E501,E221,W291,W391,E302,E251,E203,W293,E231,E303,E201,E225,E261 lib/ bin/
 
 pyflakes:
 	pyflakes lib/ansible/*.py bin/*
 
 clean:
 	@echo "Cleaning up distutils stuff"
-	-rm -rf build
-	-rm -rf dist
+	rm -rf build
+	rm -rf dist
 	@echo "Cleaning up byte compiled python stuff"
-	find . -regex ".*\.py[co]$$" -delete
+	find . -type f -regex ".*\.py[co]$$" -delete
 	@echo "Cleaning up editor backup files"
 	find . -type f \( -name "*~" -or -name "#*" \) -delete
 	find . -type f \( -name "*.swp" \) -delete
 	@echo "Cleaning up asciidoc to man transformations and results"
 	find ./docs/man -type f -name "*.xml" -delete
+	find ./docs/man -type f -name "*.asciidoc" -delete
 	@echo "Cleaning up output from test runs"
-	-rm -rf test/test_data
+	rm -rf test/test_data
 	@echo "Cleaning up RPM building stuff"
-	-rm -rf MANIFEST rpm-build
+	rm -rf MANIFEST rpm-build
+	@echo "Cleaning up Debian building stuff"
+	rm -rf debian
+	rm -rf deb-build
 
 python:
 	python setup.py build
@@ -59,7 +97,7 @@ install:
 	python setup.py install
 
 sdist: clean
-	python ./setup.py sdist
+	python setup.py sdist -t MANIFEST.in
 
 rpmcommon: sdist
 	@mkdir -p rpm-build
@@ -70,9 +108,9 @@ srpm: rpmcommon
 	--define "_builddir %{_topdir}" \
 	--define "_rpmdir %{_topdir}" \
 	--define "_srcrpmdir %{_topdir}" \
-	--define "_specdir %{_topdir}" \
+	--define "_specdir $(RPMSPECDIR)" \
 	--define "_sourcedir %{_topdir}" \
-	-bs ansible.spec
+	-bs $(RPMSPEC)
 	@echo "#############################################"
 	@echo "Ansible SRPM is built:"
 	@echo "    rpm-build/$(RPMNVR).src.rpm"
@@ -83,13 +121,21 @@ rpm: rpmcommon
 	--define "_builddir %{_topdir}" \
 	--define "_rpmdir %{_topdir}" \
 	--define "_srcrpmdir %{_topdir}" \
-	--define "_specdir %{_topdir}" \
+	--define "_specdir $(RPMSPECDIR)" \
 	--define "_sourcedir %{_topdir}" \
-	-ba ansible.spec
+	-ba $(RPMSPEC)
 	@echo "#############################################"
 	@echo "Ansible RPM is built:"
 	@echo "    rpm-build/noarch/$(RPMNVR).noarch.rpm"
 	@echo "#############################################"
 
-.PHONEY: docs manual clean pep8
-vpath %.asciidoc docs/man/man1
+debian: sdist
+deb: debian
+	cp -r packaging/debian ./
+	chmod 755 debian/rules
+	fakeroot debian/rules clean
+	fakeroot dh_install
+	fakeroot debian/rules binary
+
+# for arch or gentoo, read instructions in the appropriate 'packaging' subdirectory directory
+

README.md

@@ -31,6 +31,7 @@ Contributions to the core and modules are greatly welcome.
 
    * Required Process:
        * Submit github pull requests to the "ansible/integration" branch
+       * (This branch will be renamed 'devel' in the very near future)
        * Make sure "make tests" passes before submitting any requests.
    * Bonus points:
        * Joining the mailing list

VERSION

@@ -0,0 +1 @@
+0.3

bin/ansible

@@ -22,13 +22,13 @@
 import sys
 import getpass
 import time
-from optparse import OptionParser
 
 import ansible.runner
 import ansible.constants as C
 from ansible import utils
 from ansible import errors
 from ansible import callbacks
+from ansible import inventory
 
 ########################################################
 
@@ -46,38 +46,14 @@ class Cli(object):
 
     def parse(self):
         ''' create an options parser for bin/ansible '''
-
-        parser = OptionParser(usage = 'ansible <host-pattern> [options]')
+        
+        parser = utils.base_parser(constants=C, runas_opts=True, async_opts=True,
+            output_opts=True, connect_opts=True, usage='%prog <host-pattern> [options]')
         parser.add_option('-a', '--args', dest='module_args',
             help="module arguments", default=C.DEFAULT_MODULE_ARGS)
-        parser.add_option('-B', '--background', dest='seconds', type='int', default=0,
-            help='run asynchronously, failing after X seconds')
-        parser.add_option('-D','--debug', default=False, action="store_true",
-            help='enable standard error debugging of modules.')
-        parser.add_option('-f','--forks', dest='forks', default=C.DEFAULT_FORKS, type='int',
-            help='number of parallel processes to use')
-        parser.add_option('-i', '--inventory-file', dest='inventory',
-           help='inventory host file', default=C.DEFAULT_HOST_LIST)
-        parser.add_option('-k', '--ask-pass', default=False, action='store_true',
-            help='ask for SSH password')
-        parser.add_option('-M', '--module-path', dest='module_path',
-           help="path to module library", default=C.DEFAULT_MODULE_PATH)
         parser.add_option('-m', '--module-name', dest='module_name',
-            help="module name to execute", default=C.DEFAULT_MODULE_NAME)
-        parser.add_option('-o', '--one-line', dest='one_line', action='store_true',
-            help='condense output')
-        parser.add_option('-P', '--poll', default=C.DEFAULT_POLL_INTERVAL, type='int',
-            dest='poll_interval', help='set the poll interval if using -B')
-        parser.add_option("-s", "--sudo", default=False, action="store_true",
-            dest='sudo', help="run operations with sudo (nopasswd)")
-        parser.add_option('-t', '--tree', dest='tree', default=None,
-            help='log output to this directory')
-        parser.add_option('-T', '--timeout', default=C.DEFAULT_TIMEOUT, type='int',
-            dest='timeout', help='set the SSH timeout in seconds')
-        parser.add_option('-u', '--user', default=C.DEFAULT_REMOTE_USER,
-            dest='remote_user', help='connect as this user')
-        parser.add_option('-p', '--port', default=C.DEFAULT_REMOTE_PORT, type='int',
-            dest='remote_port', help='set the remote ssh port')
+            help="module name to execute (default=%s)" % C.DEFAULT_MODULE_NAME, 
+            default=C.DEFAULT_MODULE_NAME)
 
         options, args = parser.parse_args()
         self.callbacks.options = options        
@@ -93,13 +69,22 @@ class Cli(object):
         ''' use Runner lib to do SSH things '''
 
         pattern = args[0]
+
+        inventory_manager = inventory.Inventory(options.inventory)
+        hosts = inventory_manager.list_hosts(pattern)
+        if len(hosts) == 0:
+             print >>sys.stderr, "No hosts matched"
+             sys.exit(1) 
+        
         sshpass = None
+        sudopass = None
         if options.ask_pass:
             sshpass = getpass.getpass(prompt="SSH password: ")
-        
+        if options.ask_sudo_pass:
+            sudopass = getpass.getpass(prompt="sudo password: ")
+
         if options.tree:
             utils.prepare_writeable_dir(options.tree)
-
         if options.seconds:
             print "background launch...\n\n"
 
@@ -107,25 +92,25 @@ class Cli(object):
             module_name=options.module_name, module_path=options.module_path,
             module_args=options.module_args,
             remote_user=options.remote_user, remote_pass=sshpass,
-            host_list=options.inventory, timeout=options.timeout, 
-            remote_port=options.remote_port, forks=options.forks, 
+            inventory=inventory_manager, timeout=options.timeout, 
+            forks=options.forks, 
             background=options.seconds, pattern=pattern, 
-            callbacks=self.callbacks, sudo=options.sudo, verbose=True,
-            debug=options.debug
+            callbacks=self.callbacks, sudo=options.sudo, 
+            sudo_pass=sudopass,
+            transport=options.connection, debug=options.debug
         )
         return (runner, runner.run())
 
 
     # ----------------------------------------------
 
-    def get_polling_runner(self, old_runner, hosts, jid):
+    def get_polling_runner(self, old_runner, jid):
         return ansible.runner.Runner(
             module_name='async_status', module_path=old_runner.module_path,
             module_args="jid=%s" % jid, remote_user=old_runner.remote_user,
-            remote_pass=old_runner.remote_pass, host_list=hosts,
+            remote_pass=old_runner.remote_pass, inventory=old_runner.inventory,
             timeout=old_runner.timeout, forks=old_runner.forks,
-            remote_port=old_runner.remote_port, pattern='*', 
-            callbacks=self.silent_callbacks, verbose=True,
+            pattern='*', callbacks=self.silent_callbacks, 
         )
 
     # ----------------------------------------------
@@ -158,8 +143,10 @@ class Cli(object):
 
             clock = options.seconds
             while (clock >= 0):
-                polling_runner = self.get_polling_runner(runner, poll_hosts, jid)
+                runner.inventory.restrict_to(poll_hosts)
+                polling_runner = self.get_polling_runner(runner, jid)
                 poll_results = polling_runner.run()
+                runner.inventory.lift_restriction()
                 if poll_results is None:
                     break
                 for (host, host_result) in poll_results['contacted'].iteritems():
@@ -171,6 +158,8 @@ class Cli(object):
                 clock = clock - options.poll_interval
                 time.sleep(options.poll_interval)
                 poll_hosts = self.hosts_to_poll(poll_results)
+                if len(poll_hosts)==0:
+                    break
 
 ########################################################
 

bin/ansible-playbook

@@ -20,47 +20,34 @@
 
 import sys
 import getpass
-from optparse import OptionParser
 
 import ansible.playbook
 import ansible.constants as C
 from ansible import errors
 from ansible import callbacks
+from ansible import utils
 
 def main(args):
     ''' run ansible-playbook operations '''
 
     # create parser for CLI options
-    parser = OptionParser()
-    parser.usage = "ans-playbook playbook.yml ..."
-    parser.add_option('-D','--debug', default=False, action="store_true",
-        help='enable standard error debugging of modules.')
-    parser.add_option('-f','--forks', dest='forks', default=C.DEFAULT_FORKS, type='int',
-        help='set the number of forks to start up')
-    parser.add_option("-i", "--inventory-file", dest="inventory",
-        help="inventory host file", default=C.DEFAULT_HOST_LIST)
-    parser.add_option('-e', '--extra-vars', dest='extra_vars',
-        help='arguments to pass to the inventory script')
-    parser.add_option("-k", "--ask-pass", default=False, action="store_true",
-        help="ask for SSH password")
-    parser.add_option("-M", "--module-path", dest="module_path",
-        help="path to module library", default=C.DEFAULT_MODULE_PATH)
+    usage = "%prog playbook.yml"
+    parser = utils.base_parser(constants=C, usage=usage, connect_opts=True, runas_opts=True)
     parser.add_option('-O', '--override-hosts', dest="override_hosts", default=None,
         help="run playbook against these hosts regardless of inventory settings")
-    parser.add_option('-p', '--port', default=C.DEFAULT_REMOTE_PORT, type='int',
-        dest='remote_port', help='set the remote ssh port')
-    parser.add_option('-T', '--timeout', default=C.DEFAULT_TIMEOUT, type='int',
-        dest='timeout', help="set the SSH timeout in seconds")
 
     options, args = parser.parse_args(args)
 
     if len(args) == 0:
-        print >> sys.stderr, "playbook path is a required argument"
+        parser.print_help(file=sys.stderr)
         return 1
 
     sshpass = None
+    sudopass = None
     if options.ask_pass:
         sshpass = getpass.getpass(prompt="SSH password: ")
+    if options.ask_sudo_pass:
+        sudopass = getpass.getpass(prompt="sudo password: ")
     override_hosts = None
     if options.override_hosts:
         override_hosts = options.override_hosts.split(",")
@@ -74,19 +61,20 @@ def main(args):
 
         pb = ansible.playbook.PlayBook(
             playbook=playbook,
-            host_list=options.inventory,
-            extra_vars=options.extra_vars,
             module_path=options.module_path,
-            forks=options.forks,
-            debug=options.debug,
-            verbose=True,
-            remote_pass=sshpass,
-            remote_port=options.remote_port,
-            callbacks=playbook_cb,
-            runner_callbacks=runner_cb,
-            stats=stats,
-            timeout=options.timeout,
+            host_list=options.inventory, 
             override_hosts=override_hosts,
+            forks=options.forks, 
+            debug=options.debug, 
+            remote_user=options.remote_user,
+            remote_pass=sshpass, 
+            callbacks=playbook_cb, 
+            runner_callbacks=runner_cb, 
+            stats=stats,
+            timeout=options.timeout, 
+            transport=options.connection,
+            sudo=options.sudo,
+            sudo_pass=sudopass
         )
         try:
 

docs/man/.gitignore

@@ -1 +1,2 @@
 *.xml
+*.asciidoc

docs/man/man1/ansible-playbook.1

@@ -2,12 +2,12 @@
 .\"     Title: ansible-playbook
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/>
-.\"      Date: 04/03/2012
+.\"      Date: 04/17/2012
 .\"    Manual: System administration commands
 .\"    Source: Ansible 0.0.2
 .\"  Language: English
 .\"
-.TH "ANSIBLE\-PLAYBOOK" "1" "04/03/2012" "Ansible 0\&.0\&.2" "System administration commands"
+.TH "ANSIBLE\-PLAYBOOK" "1" "04/17/2012" "Ansible 0\&.0\&.2" "System administration commands"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -42,6 +42,10 @@ ansible\-playbook <filename\&.yml> \&... [options]
 The names of one or more YAML format files to run as ansible playbooks\&.
 .RE
 .SH "OPTIONS"
+.sp
+\fB\-D\fR, \fB\-\-debug\fR
+.sp
+Print any messages the remote module sends to standard error to the console
 .PP
 \fB\-i\fR \fIPATH\fR, \fB\-\-inventory=\fR\fIPATH\fR
 .RS 4
@@ -71,11 +75,33 @@ is specified as an integer, the default is 5\&.
 Prompt for the SSH password instead of assuming key\-based authentication with ssh\-agent\&.
 .RE
 .PP
+\fB\-K\fR, \fB\-\-ask\-sudo\-pass\fR
+.RS 4
+Prompt for the password to use for playbook plays that request sudo access, if any
+.RE
+.PP
 \fB\-T\fR \fISECONDS\fR, \fB\-\-timeout=\fR\fISECONDS\fR
 .RS 4
 Connection timeout to use when trying to talk to hosts, in
 \fISECONDS\fR\&.
 .RE
+.PP
+\fB\-O\fR \fIOVERRIDE_HOSTS\fR, \fB\-\-override\-hosts=\fR\fIOVERRIDE_HOSTS\fR
+.RS 4
+Ignore the inventory file and run the playbook against only these hosts\&. "hosts:" line in playbook should be set to
+\fIall\fR
+when using this option\&.
+.RE
+.PP
+\fB\-s\fR, \fB\-\-sudo\fR
+.RS 4
+Force all plays to use sudo, even if not marked as such\&.
+.RE
+.PP
+\fB\-u\fR \fIUSERNAME\fR, \fB\-\-remote\-user=\fR\fIUSERNAME\fR
+.RS 4
+Use this remote user name on playbook steps that do not indicate a user name to run as\&.
+.RE
 .SH "ENVIRONMENT"
 .sp
 The following environment variables may specified\&.

docs/man/man1/ansible-playbook.1.asciidoc.in

@@ -2,7 +2,7 @@ ansible-playbook(1)
 ===================
 :doctype:manpage
 :man source:   Ansible
-:man version:  0.0.2
+:man version:  %VERSION%
 :man manual:   System administration commands
 
 NAME
@@ -34,6 +34,10 @@ The names of one or more YAML format files to run as ansible playbooks.
 OPTIONS
 -------
 
+*-D*, *--debug*          
+
+Print any messages the remote module sends to standard error to the console
+
 
 *-i* 'PATH', *--inventory=*'PATH'::
 
@@ -55,11 +59,32 @@ Level of parallelism. 'NUM' is specified as an integer, the default is 5.
 Prompt for the SSH password instead of assuming key-based authentication with ssh-agent.
 
 
+*-K*, *--ask-sudo-pass*::
+
+Prompt for the password to use for playbook plays that request sudo access, if any
+
+
 *-T* 'SECONDS', *--timeout=*'SECONDS'::
 
 Connection timeout to use when trying to talk to hosts, in 'SECONDS'.
 
 
+*-O* 'OVERRIDE_HOSTS', *--override-hosts=*'OVERRIDE_HOSTS'::
+
+Ignore the inventory file and run the playbook against only these hosts.  "hosts:" line
+in playbook should be set to 'all' when using this option.
+
+
+*-s*, *--sudo*::
+
+Force all plays to use sudo, even if not marked as such.
+
+
+*-u* 'USERNAME', *--remote-user=*'USERNAME'::
+
+Use this remote user name on playbook steps that do not indicate a user name to run as.
+
+
 ENVIRONMENT
 -----------
 

docs/man/man1/ansible.1

@@ -2,12 +2,12 @@
 .\"     Title: ansible
 .\"    Author: [see the "AUTHOR" section]
 .\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/>
-.\"      Date: 04/03/2012
+.\"      Date: 04/17/2012
 .\"    Manual: System administration commands
 .\"    Source: Ansible 0.0.2
 .\"  Language: English
 .\"
-.TH "ANSIBLE" "1" "04/03/2012" "Ansible 0\&.0\&.2" "System administration commands"
+.TH "ANSIBLE" "1" "04/17/2012" "Ansible 0\&.0\&.2" "System administration commands"
 .\" -----------------------------------------------------------------
 .\" * Define some portability stuff
 .\" -----------------------------------------------------------------
@@ -79,16 +79,31 @@ The
 to pass to the module\&.
 .RE
 .PP
+\fB\-D\fR, \fB\-\-debug\fR
+.RS 4
+Print any messages the remote module sends to standard error to the console
+.RE
+.PP
 \fB\-k\fR, \fB\-\-ask\-pass\fR
 .RS 4
 Prompt for the SSH password instead of assuming key\-based authentication with ssh\-agent\&.
 .RE
 .PP
+\fB\-K\fR, \fB\-\-ask\-sudo\-pass\fR
+.RS 4
+Prompt for the password to use with \-\-sudo, if any
+.RE
+.PP
 \fB\-o\fR, \fB\-\-one\-line\fR
 .RS 4
 Try to output everything on one line\&.
 .RE
 .PP
+\fB\-s\fR, \fB\-\-sudo\fR
+.RS 4
+Run the command as the user given by \-u and sudo to root\&.
+.RE
+.PP
 \fB\-t\fR \fIDIRECTORY\fR, \fB\-\-tree=\fR\fIDIRECTORY\fR
 .RS 4
 Save contents in this output
@@ -122,6 +137,14 @@ Use this remote
 \fIUSERNAME\fR
 instead of root\&.
 .RE
+.PP
+\fB\-c\fR \fICONNECTION\fR, \fB\-\-connection=\fR\fICONNECTION\fR
+.RS 4
+Connection type to use\&. Possible options are
+\fIparamiko\fR
+(SSH) and
+\fIlocal\fR\&. Local is mostly useful for crontab or kickstarts\&.
+.RE
 .SH "INVENTORY"
 .sp
 Ansible stores the hosts it can potentially operate on in an inventory file\&. The syntax is one host per line\&. Groups headers are allowed and are included on their own line, enclosed in square brackets\&.

docs/man/man1/ansible.1.asciidoc.in

@@ -2,7 +2,7 @@ ansible(1)
 =========
 :doctype:manpage
 :man source:   Ansible
-:man version:  0.0.2
+:man version:  %VERSION%
 :man manual:   System administration commands
 
 NAME
@@ -60,42 +60,52 @@ The 'DIRECTORY' to load modules from. The default is '/usr/share/ansible'.
 
 The 'ARGUMENTS' to pass to the module.
 
+*-D*, *--debug*::
+
+Print any messages the remote module sends to standard error to the console
 
 *-k*, *--ask-pass*::
 
 Prompt for the SSH password instead of assuming key-based authentication with ssh-agent.
 
+*-K*, *--ask-sudo-pass*::
+
+Prompt for the password to use with --sudo, if any
 
 *-o*, *--one-line*::
 
 Try to output everything on one line.
 
+*-s*, *--sudo*::
+
+Run the command as the user given by -u and sudo to root.
 
 *-t* 'DIRECTORY', *--tree=*'DIRECTORY'::
 
 Save contents in this output 'DIRECTORY', with the results saved in a
 file named after each host.
 
-
 *-T* 'SECONDS', *--timeout=*'SECONDS'::
 
 Connection timeout to use when trying to talk to hosts, in 'SECONDS'.
 
-
 *-B* 'NUM', *--background=*'NUM'::
 
 Run commands in the background, killing the task after 'NUM' seconds.
 
-
 *-P* 'NUM', *--poll=*'NUM'::
 
 Poll a background job every 'NUM' seconds. Requires *-B*.
 
-
 *-u* 'USERNAME', *--remote-user=*'USERNAME'::
 
 Use this remote 'USERNAME' instead of root.
 
+*-c* 'CONNECTION', *--connection=*'CONNECTION'::
+
+Connection type to use.  Possible options are 'paramiko' (SSH) and 'local'.
+Local is mostly useful for crontab or kickstarts.
+
 
 INVENTORY
 ---------

examples/playbooks/file_secontext.yml

@@ -0,0 +1,18 @@
+---
+# This is a demo of how to manage the selinux context using the file module
+- hosts: test
+  user: root
+  tasks:
+    - name: Change setype of /etc/exports to non-default value
+      action: file path=/etc/exports setype=etc_t
+    - name: Change seuser of /etc/exports to non-default value
+      action: file path=/etc/exports seuser=unconfined_u
+    - name: Set selinux context back to default value
+      action: file path=/etc/exports context=default
+    - name: Create empty file
+      action: command /bin/touch /tmp/foo
+    - name: Change setype of /tmp/foo
+      action: file path=/tmp/foo setype=default_t
+    - name: Try to set secontext to default, but this will fail
+            because of the lack of a default in the policy
+      action: file path=/tmp/foo context=default

examples/playbooks/loop_with_items.yml

@@ -0,0 +1,30 @@
+---
+# this is an example of how to run repeated task elements over lists
+# of items, for example, installing multiple packages or configuring
+# multiple users
+
+- hosts: all
+  user: root
+
+  tasks:
+
+  - name: install $item
+    action: yum pkg=$item state=installed
+    with_items:
+       - cobbler
+       - httpd
+
+  - name: configure user $item
+    action: user name=$item state=present groups=wheel
+    with_items:
+       - testuser1
+       - testuser2
+
+  - name: remove user $item
+    action: user name=$item state=absent
+    with_items:
+        - testuser1
+        - testuser2
+
+
+

hacking/README

@@ -6,6 +6,3 @@ To use it from the root of a checkout:
     $ . ./hacking/env-setup
 
 Note the space between the '.' and the './'
-
-Man pages will not load until you run 'make docs' from the root of the
-checkout.

hacking/env-setup

@@ -4,14 +4,17 @@
 
 PREFIX_PYTHONPATH="$PWD/lib"
 PREFIX_PATH="$PWD/bin"
+PREFIX_MANPATH="$PWD/docs/man"
 
 export PYTHONPATH=$PREFIX_PYTHONPATH:$PYTHONPATH
 export PATH=$PREFIX_PATH:$PATH
 export ANSIBLE_LIBRARY="$PWD/library"
+export MANPATH=$PREFIX_MANPATH:$MANPATH
 
 echo "PATH=$PATH"
 echo "PYTHONPATH=$PYTHONPATH"
 echo "ANSIBLE_LIBRARY=$ANSIBLE_LIBRARY"
+echo "MANPATH=$MANPATH"
 
-echo "reminder: specify your host file with -i"
-echo "done."
+echo "Reminder: specify your host file with -i"
+echo "Done."

hacking/test-module

@@ -30,7 +30,7 @@ import sys
 import os
 import subprocess
 import traceback
-import ansible.utils
+from ansible import utils
 
 try:
     import json
@@ -70,7 +70,7 @@ try:
     print "***********************************"
     print "RAW OUTPUT"
     print out
-    results = ansible.utils.parse_json(out)
+    results = utils.parse_json(out)
     
 except:
     print "***********************************"
@@ -82,7 +82,7 @@ except:
 print "***********************************"
 print "PARSED OUTPUT"
 
-print results
+print utils.bigjson(results)
 
 sys.exit(0)
 

lib/ansible/__init__.py

@@ -14,5 +14,5 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-__version__ = '0.0.2'
+__version__ = '0.3'
 __author__ = 'Michael DeHaan'

lib/ansible/callbacks.py

@@ -95,6 +95,9 @@ class DefaultRunnerCallbacks(object):
     def on_unreachable(self, host, res):
         pass
 
+    def on_no_hosts(self):
+        pass
+
 ########################################################################
 
 class CliRunnerCallbacks(DefaultRunnerCallbacks):
@@ -120,6 +123,9 @@ class CliRunnerCallbacks(DefaultRunnerCallbacks):
 
     def on_error(self, host, err):
         print >>sys.stderr, "stderr: [%s] => %s\n" % (host, err)
+    
+    def on_no_hosts(self):
+        print >>sys.stderr, "no hosts matched\n"
 
     def _on_any(self, host, result):
         print utils.host_report_msg(host, self.options.module_name, result, self.options.one_line)
@@ -145,7 +151,7 @@ class PlaybookRunnerCallbacks(DefaultRunnerCallbacks):
             print "failed: [%s] => %s => %s\n" % (host, invocation, utils.smjson(results))
 
     def on_ok(self, host, host_result):
-        invocation = host_result.get('invocation',None)
+        invocation = host_result.get('invocation','')
         if invocation.startswith('async_status'):
             pass
         elif not invocation or invocation.startswith('setup '):
@@ -159,6 +165,9 @@ class PlaybookRunnerCallbacks(DefaultRunnerCallbacks):
     def on_skipped(self, host):
         print "skipping: [%s]\n" % host
 
+    def on_no_hosts(self):
+        print "no hosts matched or remaining\n"
+
 ########################################################################
 
 class PlaybookCallbacks(object):

lib/ansible/connection.py

@@ -18,11 +18,20 @@
 
 ################################################
 
-import paramiko
+import warnings
+# prevent paramiko warning noise
+# see http://stackoverflow.com/questions/3920502/
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    import paramiko
+
 import traceback
 import os
 import time
 import random
+import re
+import shutil
+import subprocess
 from ansible import errors
 
 ################################################
@@ -30,14 +39,18 @@ from ansible import errors
 class Connection(object):
     ''' Handles abstract connections to remote hosts '''
 
+    _LOCALHOSTRE = re.compile(r"^(127.0.0.1|localhost|%s)$" % os.uname()[1])
+
     def __init__(self, runner, transport):
         self.runner = runner
         self.transport = transport
 
-    def connect(self, host):
+    def connect(self, host, port=None):
         conn = None
-        if self.transport == 'paramiko':
-            conn = ParamikoConnection(self.runner, host)
+        if self.transport == 'local' and self._LOCALHOSTRE.search(host):
+            conn = LocalConnection(self.runner, host, None)
+        elif self.transport == 'paramiko':
+            conn = ParamikoConnection(self.runner, host, port)
         if conn is None:
             raise Exception("unsupported connection type")
         return conn.connect()
@@ -51,10 +64,13 @@ class Connection(object):
 class ParamikoConnection(object):
     ''' SSH based connections with Paramiko '''
 
-    def __init__(self, runner, host):
+    def __init__(self, runner, host, port=None):
         self.ssh = None
         self.runner = runner
         self.host = host
+        self.port = port
+        if port is None:
+            self.port = self.runner.remote_port
 
     def _get_conn(self):
         ssh = paramiko.SSHClient()
@@ -62,9 +78,13 @@ class ParamikoConnection(object):
 
         try:
             ssh.connect(
-                self.host, username=self.runner.remote_user,
-                allow_agent=True, look_for_keys=True, password=self.runner.remote_pass,
-                timeout=self.runner.timeout, port=self.runner.remote_port
+                self.host, 
+                username=self.runner.remote_user,
+                allow_agent=True, 
+                look_for_keys=True, 
+                password=self.runner.remote_pass,
+                timeout=self.runner.timeout, 
+                port=self.port
             )
         except Exception, e:
             if str(e).find("PID check failed") != -1:
@@ -99,15 +119,26 @@ class ParamikoConnection(object):
             sudo_chan = ssh_sudo.invoke_shell()
             sudo_chan.send("sudo -s\n")
 
+            # FIXME: using sudo with a password adds more delay, someone may wish
+            # to optimize to see when the channel is actually ready
+            if self.runner.sudo_pass:
+                time.sleep(0.1) # this is conservative
+                sudo_chan.send("%s\n" % self.runner.sudo_pass)
+                time.sleep(0.1)
+
             # to avoid ssh expect logic, redirect output to file and move the
             # file when we are done with it...
             sudo_chan.send("(%s >%s_pre 2>/dev/null ; mv %s_pre %s) &\n" % (cmd, result_file, result_file, result_file))
+            # FIXME: someone may wish to optimize to not background the launch, and tell when the command
+            # returns, removing the time.sleep(1) here
             time.sleep(1)
             sudo_chan.close()
             self.ssh = self._get_conn()
 
             # now load the results of the JSON execution...
             # FIXME: really need some timeout logic here
+            # though it doesn't make since to use the SSH timeout or impose any particular
+            # limit.  Upgrades welcome.
             sftp = self.ssh.open_sftp()
             while True:
                 # print "waiting on %s" % result_file
@@ -135,6 +166,15 @@ class ParamikoConnection(object):
             raise errors.AnsibleError("failed to transfer file to %s" % out_path)
         sftp.close()
 
+    def fetch_file(self, in_path, out_path):
+        sftp = self.ssh.open_sftp()
+        try:
+            sftp.get(in_path, out_path)
+        except IOError:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to transfer file from %s" % in_path)
+        sftp.close()
+
     def close(self):
         ''' terminate the connection '''
 
@@ -143,4 +183,52 @@ class ParamikoConnection(object):
 ############################################
 # add other connection types here
 
+class LocalConnection(object):
+    ''' Local based connections '''
+
+    def __init__(self, runner, host):
+        self.runner = runner
+        self.host = host
+
+    def connect(self, port=None):
+        ''' connect to the local host; nothing to do here '''
+
+        return self
+
+    def exec_command(self, cmd, tmp_path, sudoable=False):
+        ''' run a command on the local host '''
+        if self.runner.sudo and sudoable:
+            cmd = "sudo -s %s" % cmd
+        if self.runner.sudo_pass:
+            # NOTE: if someone wants to add sudo w/ password to the local connection type, they are welcome
+            # to do so.  The primary usage of the local connection is for crontab and kickstart usage however
+            # so this doesn't seem to be a huge priority
+            raise errors.AnsibleError("sudo with password is presently only supported on the paramiko (SSH) connection type")
+
+        p = subprocess.Popen(cmd, shell=True, stdin=None,
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        return ("", stdout, stderr)
+
+    def put_file(self, in_path, out_path):
+        ''' transfer a file from local to local '''
+        if not os.path.exists(in_path):
+            raise errors.AnsibleFileNotFound("file or module does not exist: %s" % in_path)
+        try:
+            shutil.copyfile(in_path, out_path)
+        except shutil.Error:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to copy: %s and %s are the same" % (in_path, out_path))
+        except IOError:
+            traceback.print_exc()
+            raise errors.AnsibleError("failed to transfer file to %s" % out_path)
+
+    def fetch_file(self, in_path, out_path):
+        ''' fetch a file from local to local -- for copatibility '''
+        self.put_file(in_path, out_path)
+
+    def close(self):
+        ''' terminate the connection; nothing to do here '''
+
+        pass
 

lib/ansible/constants.py

@@ -19,11 +19,11 @@
 import os
 
 
-# control side (aka 'overlord')
 DEFAULT_HOST_LIST      = os.environ.get('ANSIBLE_HOSTS',
-                                        '/etc/ansible/hosts')
+    '/etc/ansible/hosts')
 DEFAULT_MODULE_PATH    = os.environ.get('ANSIBLE_LIBRARY',
-                                        '/usr/share/ansible')
+    '/usr/share/ansible')
+
 DEFAULT_MODULE_NAME    = 'command'
 DEFAULT_PATTERN        = '*'
 DEFAULT_FORKS          = 5
@@ -32,4 +32,8 @@ DEFAULT_TIMEOUT        = 10
 DEFAULT_POLL_INTERVAL  = 15
 DEFAULT_REMOTE_USER    = 'root'
 DEFAULT_REMOTE_PASS    = None
+DEFAULT_SUDO_PASS      = None
 DEFAULT_REMOTE_PORT    = 22
+DEFAULT_TRANSPORT      = 'paramiko'
+DEFAULT_TRANSPORT_OPTS = ['local', 'paramiko']
+

lib/ansible/inventory.py

@@ -0,0 +1,292 @@
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+#############################################
+
+import fnmatch
+import os
+import subprocess
+
+import constants as C
+from ansible import errors
+from ansible import utils
+
+class Inventory(object):
+    """ Host inventory for ansible.
+
+    The inventory is either a simple text file with systems and [groups] of 
+    systems, or a script that will be called with --list or --host.
+    """
+
+    def __init__(self, host_list=C.DEFAULT_HOST_LIST):
+
+        self._restriction = None
+        self._variables = {}
+
+        if type(host_list) == list:
+            self.host_list = host_list
+            self.groups = dict(ungrouped=host_list)
+            self._is_script = False
+            return
+
+        inventory_file = os.path.expanduser(host_list)
+        if not os.path.exists(inventory_file):
+            raise errors.AnsibleFileNotFound("inventory file not found: %s" % host_list)
+
+        self.inventory_file = os.path.abspath(inventory_file)
+
+        if os.access(self.inventory_file, os.X_OK):
+            self.host_list, self.groups = self._parse_from_script()
+            self._is_script = True
+        else:
+            self.host_list, self.groups = self._parse_from_file()
+            self._is_script = False
+
+    # *****************************************************
+    # Public API
+
+    def list_hosts(self, pattern="all"):
+        """ Return a list of hosts [matching the pattern] """
+        if self._restriction is None:
+            host_list = self.host_list
+        else:
+            host_list = [ h for h in self.host_list if h in self._restriction ]
+        return [ h for h in host_list if self._matches(h, pattern) ]
+
+    def restrict_to(self, restriction):
+        """ Restrict list operations to the hosts given in restriction """
+        if type(restriction)!=list:
+            restriction = [ restriction ]
+
+        self._restriction = restriction
+
+    def lift_restriction(self):
+        """ Do not restrict list operations """
+        self._restriction = None
+
+    def get_variables(self, host):
+        """ Return the variables associated with this host. """
+
+        if host in self._variables:
+            return self._variables[host].copy()
+
+        if not self._is_script:
+            return {}
+
+        return self._get_variables_from_script(host)
+
+    # *****************************************************
+
+    def _parse_from_file(self):
+        ''' parse a textual host file '''
+
+        results = []
+        groups = dict(ungrouped=[])
+        lines = file(self.inventory_file).read().split("\n")
+        if "---" in lines:
+            return self._parse_yaml()
+        group_name = 'ungrouped'
+        for item in lines:
+            item = item.lstrip().rstrip()
+            if item.startswith("#"):
+                # ignore commented out lines
+                pass
+            elif item.startswith("["):
+                # looks like a group
+                group_name = item.replace("[","").replace("]","").lstrip().rstrip()
+                groups[group_name] = []
+            elif item != "":
+                # looks like a regular host
+                if ":" in item:
+                    # a port was specified
+                    item, port = item.split(":")
+                    try:
+                        port = int(port)
+                    except ValueError:
+                        raise errors.AnsibleError("SSH port for %s in inventory (%s) should be numerical."%(item, port))
+                    self._set_variable(item, "ansible_ssh_port", port)
+                groups[group_name].append(item)
+                if not item in results:
+                    results.append(item)
+        return (results, groups)
+
+    # *****************************************************
+
+    def _parse_from_script(self):
+        ''' evaluate a script that returns list of hosts by groups '''
+
+        results = []
+        groups = dict(ungrouped=[])
+
+        cmd = [self.inventory_file, '--list']
+
+        cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
+        out, err = cmd.communicate()
+        rc = cmd.returncode
+        if rc:
+            raise errors.AnsibleError("%s: %s" % self.inventory_file, err)
+
+        try:
+            groups = utils.json_loads(out)
+        except:
+            raise errors.AnsibleError("invalid JSON response from script: %s" % self.inventory_file)
+
+        for (groupname, hostlist) in groups.iteritems():
+            for host in hostlist:
+                if host not in results:
+                    results.append(host)
+        return (results, groups)
+
+    # *****************************************************
+
+    def _parse_yaml(self):
+        """ Load the inventory from a yaml file.
+
+        returns hosts and groups"""
+        data = utils.parse_yaml_from_file(self.inventory_file)
+
+        if type(data) != list:
+            raise errors.AnsibleError("YAML inventory should be a list.")
+
+        hosts = []
+        groups = {}
+
+        ungrouped = []
+
+        for item in data:
+            if type(item) == dict:
+                if "group" in item:
+                    group_name = item["group"]
+
+                    group_vars = []
+                    if "vars" in item:
+                        group_vars = item["vars"]
+
+                    group_hosts = []
+                    if "hosts" in item:
+                        for host in item["hosts"]:
+                            host_name = self._parse_yaml_host(host, group_vars)
+                            group_hosts.append(host_name)
+
+                    groups[group_name] = group_hosts
+                    hosts.extend(group_hosts)
+
+                elif "host" in item:
+                    host_name = self._parse_yaml_host(item)
+                    hosts.append(host_name)
+                    ungrouped.append(host_name)
+            else:
+                host_name = self._parse_yaml_host(item)
+                hosts.append(host_name)
+                ungrouped.append(host_name)
+
+        # filter duplicate hosts
+        output_hosts = []
+        for host in hosts:
+            if host not in output_hosts:
+                output_hosts.append(host)
+
+        if len(ungrouped) > 0 :
+            # hosts can be defined top-level, but also in a group
+            really_ungrouped = []
+            for host in ungrouped:
+                already_grouped = False
+                for name, group_hosts in groups.items():
+                    if host in group_hosts:
+                        already_grouped = True
+                if not already_grouped:
+                    really_ungrouped.append(host)
+            groups["ungrouped"] = really_ungrouped
+
+        return output_hosts, groups
+
+    def _parse_yaml_host(self, item, variables=[]):
+        def set_variables(host, variables):
+            if type(variables) == list:
+                for variable in variables:
+                    if len(variable) != 1:
+                        raise errors.AnsibleError("Only one item expected in %s"%(variable))
+                    k, v = variable.items()[0]
+                    self._set_variable(host, k, v)
+            elif type(variables) == dict:
+                for k, v in variables.iteritems():
+                    self._set_variable(host, k, v)
+
+
+        if type(item) in [str, unicode]:
+            set_variables(item, variables)
+            return item
+        elif type(item) == dict:
+            if "host" in item:
+                host_name = item["host"]
+                set_variables(host_name, variables)
+
+                if "vars" in item:
+                    set_variables(host_name, item["vars"])
+
+                return host_name
+        else:
+            raise errors.AnsibleError("Unknown item in inventory: %s"%(item))
+
+
+    def _get_variables_from_script(self, host):
+        ''' support per system variabes from external variable scripts, see web docs '''
+
+        cmd = [self.inventory_file, '--host', host]
+
+        cmd = subprocess.Popen(cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=False
+        )
+        out, err = cmd.communicate()
+
+        variables = {}
+        try:
+            variables = utils.json_loads(out)
+        except:
+            raise errors.AnsibleError("%s returned invalid result when called with hostname %s" % (
+                self.inventory_file,
+                host
+            ))
+        return variables
+
+    def _set_variable(self, host, key, value):
+        if not host in self._variables:
+            self._variables[host] = {}
+        self._variables[host][key] = value
+
+    def _matches(self, host_name, pattern):
+        ''' returns if a hostname is matched by the pattern '''
+
+        # a pattern is in fnmatch format but more than one pattern
+        # can be strung together with semicolons. ex:
+        #   atlanta-web*.example.com;dc-web*.example.com
+
+        if host_name == '':
+            return False
+        pattern = pattern.replace(";",":")
+        subpatterns = pattern.split(":")
+        for subpattern in subpatterns:
+            if subpattern == 'all':
+                return True
+            if fnmatch.fnmatch(host_name, subpattern):
+                return True
+            elif subpattern in self.groups:
+                if host_name in self.groups[subpattern]:
+                    return True
+        return False

lib/ansible/playbook.py

@@ -17,11 +17,11 @@
 
 #############################################
 
+import ansible.inventory
 import ansible.runner
 import ansible.constants as C
 from ansible import utils
 from ansible import errors
-import shlex
 import os
 import time
 
@@ -54,39 +54,62 @@ class PlayBook(object):
         timeout          = C.DEFAULT_TIMEOUT,
         remote_user      = C.DEFAULT_REMOTE_USER,
         remote_pass      = C.DEFAULT_REMOTE_PASS,
+        sudo_pass        = C.DEFAULT_SUDO_PASS,
         remote_port      = C.DEFAULT_REMOTE_PORT,
+        transport        = C.DEFAULT_TRANSPORT,
         override_hosts   = None,
-        extra_vars       = None,
         debug            = False,
-        verbose          = False,
         callbacks        = None,
         runner_callbacks = None,
-        stats            = None):
+        stats            = None,
+        sudo             = False):
+
+        """
+        playbook:         path to a playbook file
+        host_list:        path to a file like /etc/ansible/hosts
+        module_path:      path to ansible modules, like /usr/share/ansible/
+        forks:            desired level of paralellism
+        timeout:          connection timeout
+        remote_user:      run as this user if not specified in a particular play
+        remote_pass:      use this remote password (for all plays) vs using SSH keys
+        sudo_pass:        if sudo==True, and a password is required, this is the sudo password
+        remote_port:      default remote port to use if not specified with the host or play
+        transport:        how to connect to hosts that don't specify a transport (local, paramiko, etc)
+        override_hosts:   skip the inventory file, just talk to these hosts
+        callbacks         output callbacks for the playbook
+        runner_callbacks: more callbacks, this time for the runner API
+        stats:            holds aggregrate data about events occuring to each host
+        sudo:             if not specified per play, requests all plays use sudo mode
+        """
 
         if playbook is None or callbacks is None or runner_callbacks is None or stats is None:
             raise Exception('missing required arguments')
 
-        self.host_list        = host_list
         self.module_path      = module_path
         self.forks            = forks
         self.timeout          = timeout
         self.remote_user      = remote_user
         self.remote_pass      = remote_pass
         self.remote_port      = remote_port
+        self.transport        = transport
         self.debug            = debug
-        self.verbose          = verbose
         self.callbacks        = callbacks
         self.runner_callbacks = runner_callbacks
         self.override_hosts   = override_hosts
-        self.extra_vars       = extra_vars
         self.stats            = stats
+        self.sudo             = sudo
+        self.sudo_pass        = sudo_pass
 
-        self.basedir = os.path.dirname(playbook)
-        self.playbook = self._parse_playbook(playbook)
+        self.basedir          = os.path.dirname(playbook)
+        self.playbook         = self._parse_playbook(playbook)
+
+        if override_hosts is not None:
+            if type(override_hosts) != list:
+                raise errors.AnsibleError("override hosts must be a list")
+            self.inventory = ansible.inventory.Inventory(override_hosts)
+        else:
+            self.inventory = ansible.inventory.Inventory(host_list)
 
-        self.host_list, self.groups = ansible.runner.Runner.parse_hosts(
-            host_list, override_hosts=self.override_hosts, extra_vars=self.extra_vars)
-   
     # *****************************************************
 
     def _get_vars(self, play, dirname):
@@ -94,8 +117,18 @@ class PlayBook(object):
         if play.get('vars') is None:
             play['vars'] = {}
         vars = play['vars']
-        if type(vars) != dict:
+        if type(vars) not in [dict, list]:
             raise errors.AnsibleError("'vars' section must contain only key/value pairs")
+
+        # translate a list of vars into a dict
+        if type(vars) == list:
+            varlist = vars
+            vars = {}
+            for item in varlist:
+                k, v = item.items()[0]
+                vars[k] = v
+            play['vars'] = vars
+
         vars_prompt = play.get('vars_prompt', {})
         if type(vars_prompt) != dict:
             raise errors.AnsibleError("'vars_prompt' section must contain only key/value pairs")
@@ -113,14 +146,19 @@ class PlayBook(object):
         # include: some.yml a=2 b=3 c=4
         include_tokens = task['include'].split()
         path = utils.path_dwim(dirname, include_tokens[0])
-        inject_vars = self._get_vars(play, dirname)
+        play_vars = self._get_vars(play, dirname)
+        include_vars = {}
         for i,x in enumerate(include_tokens):
             if x.find("=") != -1:
                 (k,v) = x.split("=")
-                inject_vars[k] = v
+                include_vars[k] = v
+        inject_vars = play_vars.copy()
+        inject_vars.update(include_vars)
         included = utils.template_from_file(path, inject_vars)
         included = utils.parse_yaml(included)
         for x in included:
+            if len(include_vars):
+                x["vars"] = include_vars
             new_tasks.append(x)
 
     # *****************************************************
@@ -154,7 +192,30 @@ class PlayBook(object):
                     self._include_tasks(play, task, dirname, new_tasks)
                 else:
                     new_tasks.append(task)
-            play['tasks'] = new_tasks
+
+            # now new_tasks contains a list of tasks, but tasks may contain
+            # lists of with_items to loop over.  Do that.
+            # TODO: refactor into subfunction
+            new_tasks2 = []
+            for task in new_tasks:
+                if 'with_items' in task:
+                    for item in task['with_items']:
+                        produced_task = {}
+                        name    = task.get('name', task.get('action', 'unnamed task'))
+                        action  = task.get('action', None)
+                        only_if = task.get('only_if', None)
+                        if action is None:
+                            raise errors.AnsibleError('action is required')
+                        produced_task = task.copy()
+                        produced_task['action'] = utils.template(action, dict(item=item), SETUP_CACHE)
+                        produced_task['name'] = utils.template(name, dict(item=item), SETUP_CACHE)
+                        if only_if:
+                            produced_task['only_if'] = utils.template(only_if, dict(item=item), SETUP_CACHE)
+                        new_tasks2.append(produced_task)
+                else:
+                    new_tasks2.append(task)
+
+            play['tasks'] = new_tasks2
 
             # process handlers as well as imported handlers
             new_handlers = [] 
@@ -201,7 +262,6 @@ class PlayBook(object):
     def _async_poll(self, runner, hosts, async_seconds, async_poll_interval, only_if):
         ''' launch an async job, if poll_interval is set, wait for completion '''
 
-        runner.host_list = hosts
         runner.background = async_seconds
         results = runner.run()
         self.stats.compute(results, poll=True)
@@ -225,7 +285,7 @@ class PlayBook(object):
             return results
 
         clock = async_seconds
-        runner.host_list = self.hosts_to_poll(results)
+        host_list = self.hosts_to_poll(results)
 
         poll_results = results
         while (clock >= 0):
@@ -235,11 +295,13 @@ class PlayBook(object):
             runner.module_name = 'async_status'
             runner.background  = 0  
             runner.pattern     = '*'
+            self.inventory.restrict_to(host_list)
             poll_results       = runner.run()
             self.stats.compute(poll_results, poll=True)
-            runner.host_list       = self.hosts_to_poll(poll_results)
+            host_list          = self.hosts_to_poll(poll_results)
+            self.inventory.lift_restriction()
 
-            if len(runner.host_list) == 0:
+            if len(host_list) == 0:
                 break
             if poll_results is None:
                 break
@@ -266,32 +328,40 @@ class PlayBook(object):
 
     # *****************************************************
 
-    def _run_module(self, pattern, host_list, module, args, remote_user, 
-        async_seconds, async_poll_interval, only_if, sudo):
+    def _run_module(self, pattern, module, args, vars, remote_user, 
+        async_seconds, async_poll_interval, only_if, sudo, transport, port):
         ''' run a particular module step in a playbook '''
 
-        hosts = [ h for h in host_list if (h not in self.stats.failures) and (h not in self.stats.dark)]
+        hosts = [ h for h in self.inventory.list_hosts() if (h not in self.stats.failures) and (h not in self.stats.dark)]
+        self.inventory.restrict_to(hosts)
+
+        if port is None:
+            port=self.remote_port
 
         runner = ansible.runner.Runner(
-            pattern=pattern, groups=self.groups, module_name=module,
-            module_args=args, host_list=hosts, forks=self.forks,
+            pattern=pattern, inventory=self.inventory, module_name=module,
+            module_args=args, forks=self.forks,
             remote_pass=self.remote_pass, module_path=self.module_path,
             timeout=self.timeout, remote_user=remote_user, 
-            remote_port=self.remote_port,
+            remote_port=port, module_vars=vars,
             setup_cache=SETUP_CACHE, basedir=self.basedir,
             conditional=only_if, callbacks=self.runner_callbacks, 
-            extra_vars=self.extra_vars, debug=self.debug, sudo=sudo
+            debug=self.debug, sudo=sudo,
+            transport=transport, sudo_pass=self.sudo_pass, is_playbook=True
         )
 
         if async_seconds == 0:
-            return runner.run()
+            results = runner.run()
         else:
-            return self._async_poll(runner, hosts, async_seconds, async_poll_interval, only_if)
+            results = self._async_poll(runner, hosts, async_seconds, async_poll_interval, only_if)
+
+        self.inventory.lift_restriction()
+        return results
 
     # *****************************************************
 
-    def _run_task(self, pattern=None, host_list=None, task=None, 
-        remote_user=None, handlers=None, conditional=False, sudo=False):
+    def _run_task(self, pattern=None, task=None, 
+        remote_user=None, handlers=None, conditional=False, sudo=False, transport=None, port=None):
         ''' run a single task in the playbook and recursively run any subtasks.  '''
 
         # load the module name and parameters from the task entry
@@ -307,7 +377,12 @@ class PlayBook(object):
 
         tokens = action.split(None, 1)
         module_name = tokens[0]
-        module_args = tokens[1]
+        module_args = ''
+        if len(tokens) > 1:
+            module_args = tokens[1]
+
+        # include task specific vars
+        module_vars = task.get('vars')
 
         # tasks can be direct (run on all nodes matching
         # the pattern) or conditional, where they ran
@@ -318,9 +393,9 @@ class PlayBook(object):
 
         # load up an appropriate ansible runner to
         # run the task in parallel
-        results = self._run_module(pattern, host_list, module_name, 
-            module_args, remote_user, async_seconds, 
-            async_poll_interval, only_if, sudo)
+        results = self._run_module(pattern, module_name, 
+            module_args, module_vars, remote_user, async_seconds, 
+            async_poll_interval, only_if, sudo, transport, port)
 
         self.stats.compute(results)
 
@@ -370,7 +445,7 @@ class PlayBook(object):
 
     # *****************************************************
 
-    def _do_conditional_imports(self, vars_files, host_list):
+    def _do_conditional_imports(self, vars_files):
         ''' handle the vars_files section, which can contain variables '''
         
         # FIXME: save parsed variable results in memory to avoid excessive re-reading/parsing
@@ -381,7 +456,7 @@ class PlayBook(object):
  
         if type(vars_files) != list:
             raise errors.AnsibleError("vars_files must be a list")
-        for host in host_list:
+        for host in self.inventory.list_hosts():
             cache_vars = SETUP_CACHE.get(host,{}) 
             SETUP_CACHE[host] = cache_vars
             for filename in vars_files:
@@ -390,7 +465,7 @@ class PlayBook(object):
                     found = False
                     sequence = []
                     for real_filename in filename:
-                        filename2 = utils.path_dwim(self.basedir, utils.template(real_filename, cache_vars))
+                        filename2 = utils.path_dwim(self.basedir, utils.template(real_filename, cache_vars, SETUP_CACHE))
                         sequence.append(filename2)
                         if os.path.exists(filename2):
                             found = True
@@ -406,7 +481,7 @@ class PlayBook(object):
                         )
 
                 else:
-                    filename2 = utils.path_dwim(self.basedir, utils.template(filename, cache_vars))
+                    filename2 = utils.path_dwim(self.basedir, utils.template(filename, cache_vars, SETUP_CACHE))
                     if not os.path.exists(filename2):
                         raise errors.AnsibleError("no file matched for vars_file import: %s" % filename2)
                     data = utils.parse_yaml_from_file(filename2)
@@ -415,7 +490,7 @@ class PlayBook(object):
 
     # *****************************************************
 
-    def _do_setup_step(self, pattern, vars, user, port, sudo, vars_files=None):
+    def _do_setup_step(self, pattern, vars, user, port, sudo, transport, vars_files=None):
         ''' push variables down to the systems and get variables+facts back up '''
 
         # this enables conditional includes like $facter_os.yml and is only done
@@ -424,31 +499,29 @@ class PlayBook(object):
 
         if vars_files is not None:
             self.callbacks.on_setup_secondary()
-            self._do_conditional_imports(vars_files, self.host_list)
+            self._do_conditional_imports(vars_files)
         else:
             self.callbacks.on_setup_primary()
 
-        # first run the setup task on every node, which gets the variables
-        # written to the JSON file and will also bubble facts back up via
-        # magic in Runner()
-        push_var_str=''
-        for (k,v) in vars.iteritems():
-            push_var_str += "%s=\"%s\" " % (k,v)
-
-        host_list = [ h for h in self.host_list if not (h in self.stats.failures or h in self.stats.dark) ]
+        host_list = [ h for h in self.inventory.list_hosts(pattern) 
+                        if not (h in self.stats.failures or h in self.stats.dark) ]
+        self.inventory.restrict_to(host_list)
 
         # push any variables down to the system
         setup_results = ansible.runner.Runner(
-            pattern=pattern, groups=self.groups, module_name='setup',
-            module_args=push_var_str, host_list=host_list,
+            pattern=pattern, module_name='setup',
+            module_args=vars, inventory=self.inventory,
             forks=self.forks, module_path=self.module_path,
             timeout=self.timeout, remote_user=user,
-            remote_pass=self.remote_pass, remote_port=self.remote_port,
+            remote_pass=self.remote_pass, remote_port=port,
             setup_cache=SETUP_CACHE,
-            callbacks=self.runner_callbacks, sudo=sudo,
+            callbacks=self.runner_callbacks, sudo=sudo, debug=self.debug,
+            transport=transport, sudo_pass=self.sudo_pass, is_playbook=True
         ).run()
         self.stats.compute(setup_results, setup=True)
 
+        self.inventory.lift_restriction()
+
         # now for each result, load into the setup cache so we can
         # let runner template out future commands
         setup_ok = setup_results.get('contacted', {})
@@ -457,15 +530,6 @@ class PlayBook(object):
             for (host, result) in setup_ok.iteritems():
                 SETUP_CACHE[host] = result
 
-        if self.extra_vars:
-            extra_vars = utils.parse_kv(self.extra_vars)
-            for h in self.host_list:
-                try:
-                    SETUP_CACHE[h].update(extra_vars)
-                except:
-                    SETUP_CACHE[h] = extra_vars
-        return host_list
-
     # *****************************************************
 
     def _run_play(self, pg):
@@ -484,26 +548,28 @@ class PlayBook(object):
         handlers   = pg.get('handlers', [])
         user       = pg.get('user', self.remote_user)
         port       = pg.get('port', self.remote_port)
-        sudo       = pg.get('sudo', False)
+        sudo       = pg.get('sudo', self.sudo)
+        transport  = pg.get('connection', self.transport)
 
         self.callbacks.on_play_start(pattern)
 
         # push any variables down to the system # and get facts/ohai/other data back up
-        self._do_setup_step(pattern, vars, user, port, sudo, None)
+        self._do_setup_step(pattern, vars, user, port, sudo, transport, None)
          
         # now with that data, handle contentional variable file imports!
         if len(vars_files) > 0:
-            self._do_setup_step(pattern, vars, user, port, sudo, vars_files)
+            self._do_setup_step(pattern, vars, user, port, sudo, transport, vars_files)
 
         # run all the top level tasks, these get run on every node
         for task in tasks:
             self._run_task(
                 pattern=pattern,
-                host_list=self.host_list,
                 task=task, 
                 handlers=handlers,
                 remote_user=user,
-                sudo=sudo
+                sudo=sudo,
+                transport=transport,
+                port=port
             )
 
         # handlers only run on certain nodes, they are flagged by _flag_handlers
@@ -515,15 +581,18 @@ class PlayBook(object):
         for task in handlers:
             triggered_by = task.get('run', None)
             if type(triggered_by) == list:
+                self.inventory.restrict_to(triggered_by)
                 self._run_task(
                    pattern=pattern, 
                    task=task,
                    handlers=[],
-                   host_list=triggered_by,
                    conditional=True,
                    remote_user=user,
-                   sudo=sudo
+                   sudo=sudo,
+                   transport=transport,
+                   port=port
                 )
+                self.inventory.lift_restriction()
 
         # end of execution for this particular pattern.  Multiple patterns
         # can be in a single playbook file

lib/ansible/runner.py

@@ -18,18 +18,20 @@
 
 ################################################
 
-import fnmatch
 import multiprocessing
 import signal
 import os
+import pwd
 import Queue
 import random
 import traceback
 import tempfile
-import subprocess
+import base64
+import getpass
 
 import ansible.constants as C 
 import ansible.connection
+import ansible.inventory
 from ansible import utils
 from ansible import errors
 from ansible import callbacks as ans_callbacks
@@ -66,16 +68,41 @@ def _executor_hook(job_queue, result_queue):
 
 class Runner(object):
 
-    _external_variable_script = None
-
-    def __init__(self, host_list=C.DEFAULT_HOST_LIST, module_path=C.DEFAULT_MODULE_PATH,
+    def __init__(self, 
+        host_list=C.DEFAULT_HOST_LIST, module_path=C.DEFAULT_MODULE_PATH,
         module_name=C.DEFAULT_MODULE_NAME, module_args=C.DEFAULT_MODULE_ARGS, 
-        forks=C.DEFAULT_FORKS, timeout=C.DEFAULT_TIMEOUT, pattern=C.DEFAULT_PATTERN,
-        remote_user=C.DEFAULT_REMOTE_USER, remote_pass=C.DEFAULT_REMOTE_PASS,
-        remote_port=C.DEFAULT_REMOTE_PORT, background=0, basedir=None, setup_cache=None,
-        transport='paramiko', conditional='True', groups={}, callbacks=None, verbose=False,
-        debug=False, sudo=False, extra_vars=None):
-   
+        forks=C.DEFAULT_FORKS, timeout=C.DEFAULT_TIMEOUT, 
+        pattern=C.DEFAULT_PATTERN, remote_user=C.DEFAULT_REMOTE_USER, 
+        remote_pass=C.DEFAULT_REMOTE_PASS, remote_port=C.DEFAULT_REMOTE_PORT, 
+        sudo_pass=C.DEFAULT_SUDO_PASS, background=0, basedir=None, 
+        setup_cache=None, transport=C.DEFAULT_TRANSPORT, conditional='True', 
+        callbacks=None, debug=False, sudo=False, module_vars=None, 
+        is_playbook=False, inventory=None):
+
+        """
+        host_list    : path to a host list file, like /etc/ansible/hosts
+        module_path  : path to modules, like /usr/share/ansible
+        module_name  : which module to run (string)
+        module_args  : args to pass to the module (string)
+        forks        : desired level of paralellism (hosts to run on at a time)
+        timeout      : connection timeout, such as a SSH timeout, in seconds
+        pattern      : pattern or groups to select from in inventory
+        remote_user  : connect as this remote username
+        remote_pass  : supply this password (if not using keys)
+        remote_port  : use this default remote port (if not set by the inventory system)
+        sudo_pass    : sudo password if using sudo and sudo requires a password
+        background   : run asynchronously with a cap of this many # of seconds (if not 0)
+        basedir      : paths used by modules if not absolute are relative to here
+        setup_cache  : this is a internalism that is going away
+        transport    : transport mode (paramiko, local)
+        conditional  : only execute if this string, evaluated, is True
+        callbacks    : output callback class
+        sudo         : log in as remote user and immediately sudo to root
+        module_vars  : provides additional variables to a template.  FIXME: just use module_args, remove
+        is_playbook  : indicates Runner is being used by a playbook.  affects behavior in various ways.
+        inventory    : inventory object, if host_list is not provided
+        """
+
         if setup_cache is None:
             setup_cache = {}
         if basedir is None: 
@@ -86,13 +113,14 @@ class Runner(object):
         self.callbacks = callbacks
 
         self.generated_jid = str(random.randint(0, 999999999999))
-        self.connector = ansible.connection.Connection(self, transport)
 
-        if type(host_list) == str:
-            self.host_list, self.groups = self.parse_hosts(host_list)
+        self.transport = transport
+        self.connector = ansible.connection.Connection(self, self.transport)
+
+        if inventory is None:
+            self.inventory = ansible.inventory.Inventory(host_list)
         else:
-            self.host_list = host_list
-            self.groups    = groups
+            self.inventory = inventory
 
         self.setup_cache = setup_cache
         self.conditional = conditional
@@ -101,133 +129,39 @@ class Runner(object):
         self.forks       = int(forks)
         self.pattern     = pattern
         self.module_args = module_args
-        self.extra_vars  = extra_vars
+        self.module_vars = module_vars
         self.timeout     = timeout
         self.debug       = debug
-        self.verbose     = verbose
         self.remote_user = remote_user
         self.remote_pass = remote_pass
         self.remote_port = remote_port
         self.background  = background
         self.basedir     = basedir
         self.sudo        = sudo
+        self.sudo_pass   = sudo_pass
+        self.is_playbook = is_playbook
 
-        if type(self.module_args) != str:
-            raise Exception("module_args must be a string: %s" % self.module_args)
+        euid = pwd.getpwuid(os.geteuid())[0]
+        if self.transport == 'local' and self.remote_user != euid:
+            raise Exception("User mismatch: expected %s, but is %s" % (self.remote_user, euid))
+        if type(self.module_args) not in [str, unicode, dict]:
+            raise Exception("module_args must be a string or dict: %s" % self.module_args)
 
         self._tmp_paths  = {}
         random.seed()
 
-
     # *****************************************************
 
     @classmethod
-    def parse_hosts_from_regular_file(cls, host_list):
-        ''' parse a textual host file '''
-
-        results = []
-        groups = dict(ungrouped=[])
-        lines = file(host_list).read().split("\n")
-        group_name = 'ungrouped'
-        for item in lines:
-            item = item.lstrip().rstrip()
-            if item.startswith("#"):
-                # ignore commented out lines
-                pass
-            elif item.startswith("["):
-                # looks like a group
-                group_name = item.replace("[","").replace("]","").lstrip().rstrip()
-                groups[group_name] = []
-            elif item != "":
-                # looks like a regular host
-                groups[group_name].append(item)
-                if not item in results:
-                    results.append(item)
-        return (results, groups)
-
-    # *****************************************************
-
-    @classmethod
-    def parse_hosts_from_script(cls, host_list, extra_vars):
-        ''' evaluate a script that returns list of hosts by groups '''
-
-        results = []
-        groups = dict(ungrouped=[])
-        host_list = os.path.abspath(host_list)
-        cls._external_variable_script = host_list
-        cmd = [host_list, '--list']
-        if extra_vars:
-            cmd.extend(['--extra-vars', extra_vars])
-        cmd = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
-        out, err = cmd.communicate()
-        rc = cmd.returncode
-        if rc:
-            raise errors.AnsibleError("%s: %s" % (host_list, err))
-        try:
-            groups = utils.json_loads(out)
-        except:
-            raise errors.AnsibleError("invalid JSON response from script: %s" % host_list)
-        for (groupname, hostlist) in groups.iteritems():
-            for host in hostlist:
-                if host not in results:
-                    results.append(host)
-        return (results, groups)
-
-    # *****************************************************
-
-    @classmethod
-    def parse_hosts(cls, host_list, override_hosts=None, extra_vars=None):
+    def parse_hosts(cls, host_list, override_hosts=None):
         ''' parse the host inventory file, returns (hosts, groups) '''
 
-        if override_hosts is not None:
-            if type(override_hosts) != list:
-                raise errors.AnsibleError("override hosts must be a list")
-            return (override_hosts, dict(ungrouped=override_hosts))
-
-        if type(host_list) == list:
-            raise Exception("function can only be called on inventory files")
-
-        host_list = os.path.expanduser(host_list)
-        if not os.path.exists(host_list):
-            raise errors.AnsibleFileNotFound("inventory file not found: %s" % host_list)
-
-        if not os.access(host_list, os.X_OK):
-            return Runner.parse_hosts_from_regular_file(host_list)
+        if override_hosts is None:
+            inventory = ansible.inventory.Inventory(host_list)
         else:
-            return Runner.parse_hosts_from_script(host_list, extra_vars)
+            inventory = ansible.inventory.Inventory(override_hosts)
 
-    # *****************************************************
-
-    def _matches(self, host_name, pattern):
-        ''' returns if a hostname is matched by the pattern '''
-
-        # a pattern is in fnmatch format but more than one pattern
-        # can be strung together with semicolons. ex:
-        #   atlanta-web*.example.com;dc-web*.example.com
-
-        if host_name == '':
-            return False
-        pattern = pattern.replace(";",":")
-        subpatterns = pattern.split(":")
-        for subpattern in subpatterns:
-            if subpattern == 'all':
-                return True
-            if fnmatch.fnmatch(host_name, subpattern):
-                return True
-            elif subpattern in self.groups:
-                if host_name in self.groups[subpattern]:
-                    return True
-        return False
-
-    # *****************************************************
-
-    def _connect(self, host):
-        ''' connects to a host, returns (is_successful, connection_object OR traceback_string) '''
-
-        try:
-            return [ True, self.connector.connect(host) ]
-        except errors.AnsibleConnectionFailed, e:
-            return [ False, "FAILED: %s" % str(e) ]
+        return inventory.host_list, inventory.groups
 
     # *****************************************************
 
@@ -252,7 +186,7 @@ class Runner(object):
         if type(files) == str:
             files = [ files ]
         for filename in files:
-            if not filename.startswith('/tmp/'):
+            if filename.find('/tmp/') == -1:
                 raise Exception("not going to happen")
             self._exec_command(conn, "rm -rf %s" % filename, None)
 
@@ -267,71 +201,65 @@ class Runner(object):
 
     # *****************************************************
 
-    def _transfer_argsfile(self, conn, tmp, args_str):
-        ''' transfer arguments as a single file to be fed to the module. '''
+    def _transfer_str(self, conn, tmp, name, data):
+        ''' transfer string to remote file '''
 
-        args_fd, args_file = tempfile.mkstemp()
-        args_fo = os.fdopen(args_fd, 'w')
-        args_fo.write(args_str)
-        args_fo.flush()
-        args_fo.close()
+        if type(data) == dict:
+            data = utils.smjson(data)
 
-        args_remote = os.path.join(tmp, 'arguments')
-        conn.put_file(args_file, args_remote)
-        os.unlink(args_file)
+        afd, afile = tempfile.mkstemp()
+        afo = os.fdopen(afd, 'w')
+        afo.write(data)
+        afo.flush()
+        afo.close()
 
-        return args_remote
-
-    # *****************************************************
-
-    def _add_variables_from_script(self, conn, inject):
-        ''' support per system variabes from external variable scripts, see web docs '''
-
-        host = conn.host
-
-        cmd = [Runner._external_variable_script, '--host', host]
-        if self.extra_vars:
-            cmd.extend(['--extra-vars', self.extra_vars])
-
-        cmd = subprocess.Popen(cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            shell=False
-        )
-        out, err = cmd.communicate()
-        inject2 = {}
-        try:
-            inject2 = utils.json_loads(out)
-        except:
-            raise errors.AnsibleError("%s returned invalid result when called with hostname %s" % (
-                Runner._external_variable_script,
-                host
-            ))
-        # store injected variables in the templates
-        inject.update(inject2)
+        remote = os.path.join(tmp, name)
+        conn.put_file(afile, remote)
+        os.unlink(afile)
+        return remote
 
     # *****************************************************
 
     def _add_setup_vars(self, inject, args):
         ''' setup module variables need special handling '''
 
+        is_dict = False
+        if type(args) == dict:
+            is_dict = True
+
+        # TODO: keep this as a dict through the whole path to simplify this code
         for (k,v) in inject.iteritems():
-            if not k.startswith('facter_') and not k.startswith('ohai_'):
-                if str(v).find(" ") != -1:
-                    v = "\"%s\"" % v
-                args += " %s=%s" % (k, str(v).replace(" ","~~~"))
+            if not k.startswith('facter_') and not k.startswith('ohai_') and not k.startswith('ansible_'):
+                if not is_dict:
+                    if str(v).find(" ") != -1:
+                        v = "\"%s\"" % v
+                    args += " %s=%s" % (k, str(v).replace(" ","~~~"))
+                else:
+                    args[k]=v
         return args   
  
     # *****************************************************
 
     def _add_setup_metadata(self, args):
         ''' automatically determine where to store variables for the setup module '''
+        
+        is_dict = False
+        if type(args) == dict:
+            is_dict = True
 
-        if args.find("metadata=") == -1:
-            if self.remote_user == 'root':
-                args = "%s metadata=/etc/ansible/setup" % args
-            else:
-                args = "%s metadata=/home/%s/.ansible/setup" % (args, self.remote_user)
+        # TODO: keep this as a dict through the whole path to simplify this code
+        if not is_dict:
+            if args.find("metadata=") == -1:
+                if self.remote_user == 'root':
+                    args = "%s metadata=/etc/ansible/setup" % args
+                else:
+                    args = "%s metadata=/home/%s/.ansible/setup" % (args, self.remote_user)
+        else:
+            if not 'metadata' in args:
+                if self.remote_user == 'root':
+                    args['metadata'] = '/etc/ansible/setup'
+                else:
+                    args['metadata'] = "/home/%s/.ansible/setup" % (self.remote_user)
         return args   
  
     # *****************************************************
@@ -341,36 +269,53 @@ class Runner(object):
         ''' runs a module that has already been transferred '''
 
         inject = self.setup_cache.get(conn.host,{})
-        conditional = utils.double_template(self.conditional, inject)
+        conditional = utils.double_template(self.conditional, inject, self.setup_cache)
         if not eval(conditional):
             return [ utils.smjson(dict(skipped=True)), None, 'skipped' ]
 
-        if Runner._external_variable_script is not None:
-            self._add_variables_from_script(conn, inject)
+        host_variables = self.inventory.get_variables(conn.host)
+        inject.update(host_variables)
+
         if self.module_name == 'setup':
             args = self._add_setup_vars(inject, args)
             args = self._add_setup_metadata(args)
 
-        args = utils.template(args, inject)
-        module_name_tail = remote_module_path.split("/")[-1]
-        client_executed_str = "%s %s" % (module_name_tail, args.strip())
+        if type(args) == dict:
+            args = utils.bigjson(args)
+        args = utils.template(args, inject, self.setup_cache)
 
-        argsfile = self._transfer_argsfile(conn, tmp, args)
+        module_name_tail = remote_module_path.split("/")[-1]
+
+        argsfile = self._transfer_str(conn, tmp, 'arguments', args)
         if async_jid is None:
             cmd = "%s %s" % (remote_module_path, argsfile)
         else:
             cmd = " ".join([str(x) for x in [remote_module_path, async_jid, async_limit, async_module, argsfile]])
 
-        # log command as the full command not as the path to args file - helps with debugging
-        msg = '%s: "%s"' % (self.module_name, args)
-        conn.exec_command('/usr/bin/logger -t ansible -p auth.info "%s"' % msg, None)
-
-                    
         res, err = self._exec_command(conn, cmd, tmp, sudoable=True)
+        client_executed_str = "%s %s" % (module_name_tail, args.strip())
         return ( res, err, client_executed_str )
 
     # *****************************************************
 
+    def _save_setup_result_to_disk(self, conn, result):
+       ''' cache results of calling setup '''
+
+       dest = os.path.expanduser("~/.ansible_setup_data")
+       user = getpass.getuser()
+       if user == 'root':
+           dest = "/var/lib/ansible/setup_data"
+       if not os.path.exists(dest):
+           os.makedirs(dest)
+
+       fh = open(os.path.join(dest, conn.host), "w")
+       fh.write(result)
+       fh.close()
+
+       return result
+
+    # *****************************************************
+
     def _add_result_to_setup_cache(self, conn, result):
         ''' allows discovered variables to be used in templates and action statements '''
 
@@ -404,6 +349,8 @@ class Runner(object):
 
         if module_name == 'setup':
             self._add_result_to_setup_cache(conn, result)
+            if self.is_playbook:
+                self._save_setup_result_to_disk(conn, result)
 
         return self._return_from_module(conn, host, result, err, executed)
 
@@ -436,9 +383,15 @@ class Runner(object):
 
         # load up options
         options = utils.parse_kv(self.module_args)
-        source = options['src']
-        dest   = options['dest']
-        
+        source = options.get('src', None)
+        dest   = options.get('dest', None)
+        if source is None or dest is None:
+            return (host, True, dict(failed=True, msg="src and dest are required"), '')
+
+        # apply templating to source argument
+        inject = self.setup_cache.get(conn.host,{})
+        source = utils.template(source, inject, self.setup_cache)
+
         # transfer the file to a remote tmp location
         tmp_src = tmp + source.split('/')[-1]
         conn.put_file(utils.path_dwim(self.basedir, source), tmp_src)
@@ -459,6 +412,41 @@ class Runner(object):
 
     # *****************************************************
 
+    def _execute_fetch(self, conn, host, tmp):
+        ''' handler for fetch operations '''
+
+        # load up options
+        options = utils.parse_kv(self.module_args)
+        source = options.get('src', None)
+        dest = options.get('dest', None)
+        if source is None or dest is None:
+            return (host, True, dict(failed=True, msg="src and dest are required"), '')
+
+        # files are saved in dest dir, with a subdir for each host, then the filename
+        dest   = "%s/%s/%s" % (utils.path_dwim(self.basedir, dest), host, source)
+        dest   = dest.replace("//","/")
+
+        # compare old and new md5 for support of change hooks
+        local_md5 = None
+        if os.path.exists(dest):
+            local_md5 = os.popen("md5sum %s" % dest).read().split()[0]
+        remote_md5 = self._exec_command(conn, "md5sum %s" % source, tmp, True)[0].split()[0]
+
+        if remote_md5 != local_md5:
+            # create the containing directories, if needed
+            os.makedirs(os.path.dirname(dest))
+            # fetch the file and check for changes
+            conn.fetch_file(source, dest)
+            new_md5 = os.popen("md5sum %s" % dest).read().split()[0]
+            if new_md5 != remote_md5:
+                return (host, True, dict(failed=True, msg="md5 mismatch", md5sum=new_md5), '')
+            return (host, True, dict(changed=True, md5sum=new_md5), '')
+        else:
+            return (host, True, dict(changed=False, md5sum=local_md5), '')
+        
+        
+    # *****************************************************
+
     def _chain_file_module(self, conn, tmp, data, err, options, executed):
         ''' handles changing file attribs after copy/template operations '''
 
@@ -481,28 +469,60 @@ class Runner(object):
 
         # load up options
         options  = utils.parse_kv(self.module_args)
-        source   = options['src']
-        dest     = options['dest']
+        source   = options.get('src', None)
+        dest     = options.get('dest', None)
         metadata = options.get('metadata', None)
+        if source is None or dest is None:
+            return (host, True, dict(failed=True, msg="src and dest are required"), '')
 
-        if metadata is None:
-            if self.remote_user == 'root':
-                metadata = '/etc/ansible/setup'
-            else:
-                metadata = '~/.ansible/setup'
+        # apply templating to source argument so vars can be used in the path
+        inject = self.setup_cache.get(conn.host,{})
+        source = utils.template(source, inject, self.setup_cache)
 
-        # first copy the source template over
-        temppath = tmp + os.path.split(source)[-1]
-        conn.put_file(utils.path_dwim(self.basedir, source), temppath)
+        (host, ok, data, err) = (None, None, None, None)
+
+        if not self.is_playbook:
+
+            # not running from a playbook so we have to fetch the remote
+            # setup file contents before proceeding...
+            if metadata is None:
+                if self.remote_user == 'root':
+                    metadata = '/etc/ansible/setup'
+                else:
+                    # path is expanded on remote side
+                    metadata = "~/.ansible/setup"
+            
+            # install the template module
+            slurp_module = self._transfer_module(conn, tmp, 'slurp')
+
+            # run the slurp module to get the metadata file
+            args = "src=%s" % metadata
+            (result1, err, executed) = self._execute_module(conn, tmp, slurp_module, args)
+            result1 = utils.json_loads(result1)
+            if not 'content' in result1 or result1.get('encoding','base64') != 'base64':
+                result1['failed'] = True
+                return self._return_from_module(conn, host, result1, err, executed)
+            content = base64.b64decode(result1['content'])
+            inject = utils.json_loads(content)
 
         # install the template module
-        template_module = self._transfer_module(conn, tmp, 'template')
+        copy_module = self._transfer_module(conn, tmp, 'copy')
 
-        # run the template module
-        args = "src=%s dest=%s metadata=%s" % (temppath, dest, metadata)
-        (result1, err, executed) = self._execute_module(conn, tmp, template_module, args)
+        # template the source data locally
+        source_data = file(utils.path_dwim(self.basedir, source)).read()
+        resultant = ''            
+        try:
+            resultant = utils.template(source_data, inject, self.setup_cache)
+        except Exception, e:
+            return (host, False, dict(failed=True, msg=str(e)), '')
+        xfered = self._transfer_str(conn, tmp, 'source', resultant)
+            
+        # run the COPY module
+        args = "src=%s dest=%s" % (xfered, dest)
+        (result1, err, executed) = self._execute_module(conn, tmp, copy_module, args)
         (host, ok, data, err) = self._return_from_module(conn, host, result1, err, executed)
-
+ 
+        # modify file attribs if needed
         if ok:
             return self._chain_file_module(conn, tmp, data, err, options, executed)
         else:
@@ -528,18 +548,25 @@ class Runner(object):
     def _executor_internal(self, host):
         ''' callback executed in parallel for each host. returns (hostname, connected_ok, extra) '''
 
-        ok, conn = self._connect(host)
-        if not ok:
-            return [ host, False, conn , None]
-        
+        host_variables = self.inventory.get_variables(host)
+        port = host_variables.get('ansible_ssh_port', self.remote_port)
+
+        conn = None
+        try:
+            conn = self.connector.connect(host, port)
+        except errors.AnsibleConnectionFailed, e:
+            return [ host, False, "FAILED: %s" % str(e), None ]
+
         cache = self.setup_cache.get(host, {})
-        module_name = utils.template(self.module_name, cache)
+        module_name = utils.template(self.module_name, cache, self.setup_cache)
 
         tmp = self._get_tmp_path(conn)
         result = None
 
         if self.module_name == 'copy':
             result = self._execute_copy(conn, host, tmp)
+        elif self.module_name == 'fetch':
+            result = self._execute_fetch(conn, host, tmp)
         elif self.module_name == 'template':
             result = self._execute_template(conn, host, tmp)
         else:
@@ -562,8 +589,9 @@ class Runner(object):
             else:
                 self.callbacks.on_ok(host, data)
 
-            if self.debug and err:
-                self.callbacks.on_error(host, err)
+            if err:
+                if self.debug or data.get('parsed', True) == False:
+                    self.callbacks.on_error(host, err)
 
         return result
 
@@ -572,10 +600,6 @@ class Runner(object):
     def _exec_command(self, conn, cmd, tmp, sudoable=False):
         ''' execute a command string over SSH, return the output '''
 
-        msg = '%s: %s' % (self.module_name, cmd)
-        # log remote command execution
-        conn.exec_command('/usr/bin/logger -t ansible -p auth.info "%s"' % msg, None)
-        # now run actual command
         stdin, stdout, stderr = conn.exec_command(cmd, tmp, sudoable=sudoable)
 
         if type(stderr) != str:
@@ -593,7 +617,14 @@ class Runner(object):
     def _get_tmp_path(self, conn):
         ''' gets a temporary path on a remote box '''
 
-        result, err = self._exec_command(conn, "mktemp -d /tmp/ansible.XXXXXX", None, sudoable=False)
+        basetmp = "/var/tmp"
+        if self.remote_user != 'root':
+            basetmp = "/home/%s/.ansible/tmp" % self.remote_user
+        cmd = "mktemp -d %s/ansible.XXXXXX" % basetmp 
+        if self.remote_user != 'root':
+            cmd = "mkdir -p %s && %s" % (basetmp, cmd)
+
+        result, err = self._exec_command(conn, cmd, None, sudoable=False)
         cleaned = result.split("\n")[0].strip() + '/'
         return cleaned
 
@@ -615,13 +646,6 @@ class Runner(object):
 
     # *****************************************************
 
-    def _match_hosts(self, pattern):
-        ''' return all matched hosts fitting a pattern '''
-
-        return [ h for h in self.host_list if self._matches(h, pattern) ]
-
-    # *****************************************************
-
     def _parallel_exec(self, hosts):
         ''' handles mulitprocessing when more than 1 fork is required '''
 
@@ -668,7 +692,7 @@ class Runner(object):
                 results2["dark"][host] = result
 
         # hosts which were contacted but never got a chance to return
-        for host in self._match_hosts(self.pattern):
+        for host in self.inventory.list_hosts(self.pattern):
             if not (host in results2['dark'] or host in results2['contacted']):
                 results2["dark"][host] = {}
 
@@ -680,8 +704,9 @@ class Runner(object):
         ''' xfer & run module on all matched hosts '''
        
         # find hosts that match the pattern
-        hosts = self._match_hosts(self.pattern)
+        hosts = self.inventory.list_hosts(self.pattern)
         if len(hosts) == 0:
+            self.callbacks.on_no_hosts()
             return dict(contacted={}, dark={})
  
         hosts = [ (self,x) for x in hosts ]

lib/ansible/utils.py

@@ -23,13 +23,15 @@ import shlex
 import re
 import jinja2
 import yaml
-
+import optparse
+from operator import methodcaller
 try:
     import json
 except ImportError:
     import simplejson as json
 
 from ansible import errors
+import ansible.constants as C
 
 ###############################################################
 # UTILITY FUNCTIONS FOR COMMAND LINE TOOLS
@@ -236,14 +238,16 @@ def varReplace(raw, vars):
 
     return ''.join(done)
 
-def template(text, vars):
+def template(text, vars, setup_cache):
     ''' run a text buffer through the templating engine '''
+    vars = vars.copy()
     text = varReplace(str(text), vars)
+    vars['hostvars'] = setup_cache
     template = jinja2.Template(text)
     return template.render(vars)
 
-def double_template(text, vars):
-    return template(template(text, vars), vars)
+def double_template(text, vars, setup_cache):
+    return template(template(text, vars, setup_cache), vars, setup_cache)
 
 def template_from_file(path, vars):
     ''' run a file through the templating engine '''
@@ -270,3 +274,60 @@ def parse_kv(args):
             options[k]=v
     return options
 
+class SortedOptParser(optparse.OptionParser):
+    '''Optparser which sorts the options by opt before outputting --help'''
+    def format_help(self, formatter=None):
+        self.option_list.sort(key=methodcaller('get_opt_string'))
+        return optparse.OptionParser.format_help(self, formatter=None)
+
+def base_parser(constants=C, usage="", output_opts=False, runas_opts=False, async_opts=False, connect_opts=False):
+    ''' create an options parser for any ansible script '''
+
+    parser = SortedOptParser(usage)
+    parser.add_option('-D','--debug', default=False, action="store_true",
+        help='debug standard error output of remote modules')
+    parser.add_option('-f','--forks', dest='forks', default=constants.DEFAULT_FORKS, type='int',
+        help="specify number of parallel processes to use (default=%s)" % constants.DEFAULT_FORKS)
+    parser.add_option('-i', '--inventory-file', dest='inventory',
+        help="specify inventory host file (default=%s)" % constants.DEFAULT_HOST_LIST, 
+        default=constants.DEFAULT_HOST_LIST)
+    parser.add_option('-k', '--ask-pass', default=False, dest='ask_pass', action='store_true',
+        help='ask for SSH password')
+    parser.add_option('-K', '--ask-sudo-pass', default=False, dest='ask_sudo_pass', action='store_true',
+        help='ask for sudo password')
+    parser.add_option('-M', '--module-path', dest='module_path',
+        help="specify path to module library (default=%s)" % constants.DEFAULT_MODULE_PATH, 
+        default=constants.DEFAULT_MODULE_PATH)
+    parser.add_option('-T', '--timeout', default=constants.DEFAULT_TIMEOUT, type='int',
+        dest='timeout', 
+        help="override the SSH timeout in seconds (default=%s)" % constants.DEFAULT_TIMEOUT)
+
+    if output_opts:
+        parser.add_option('-o', '--one-line', dest='one_line', action='store_true',
+            help='condense output')
+        parser.add_option('-t', '--tree', dest='tree', default=None,
+            help='log output to this directory')
+
+    if runas_opts:
+        parser.add_option("-s", "--sudo", default=False, action="store_true",
+            dest='sudo', help="run operations with sudo (nopasswd)")
+        parser.add_option('-u', '--user', default=constants.DEFAULT_REMOTE_USER,
+            dest='remote_user', 
+            help='connect as this user (default=%s)' % constants.DEFAULT_REMOTE_USER)
+    
+    if connect_opts:
+        parser.add_option('-c', '--connection', dest='connection',
+                          choices=C.DEFAULT_TRANSPORT_OPTS,
+                          default=C.DEFAULT_TRANSPORT,
+                          help="connection type to use (default=%s)" % C.DEFAULT_TRANSPORT)
+
+    if async_opts:
+        parser.add_option('-P', '--poll', default=constants.DEFAULT_POLL_INTERVAL, type='int',
+            dest='poll_interval', 
+            help="set the poll interval if using -B (default=%s)" % constants.DEFAULT_POLL_INTERVAL)
+        parser.add_option('-B', '--background', dest='seconds', type='int', default=0,
+            help='run asynchronously, failing after X seconds (default=N/A)')
+
+    return parser
+
+

library/apt

@@ -31,9 +31,7 @@ APT_PATH = "/usr/bin/apt-get"
 APT = "DEBIAN_PRIORITY=critical %s" % APT_PATH
 
 def debug(msg):
-    # ansible ignores stderr, so it's safe to use for debug
     print >>sys.stderr, msg
-    #pass
     
 def exit_json(rc=0, **kwargs):
     print json.dumps(kwargs)
@@ -44,9 +42,9 @@ def fail_json(**kwargs):
     exit_json(rc=1, **kwargs)
 
 try:
-    import apt
+    import apt, apt_pkg
 except ImportError:
-    fail_json(msg="could not import apt")
+    fail_json(msg="could not import apt, please install the python-apt package on this host")
 
 def run_apt(command):
     try:
@@ -65,34 +63,48 @@ def run_apt(command):
         rc = cmd.returncode
         return rc, out, err
 
-def package_status(pkgspec, cache):
-    try:
-        pkg = cache[pkgspec]
-    except:
-        fail_json(msg="No package matching '%s' is available" % pkgspec)
-    return (pkg.is_installed, pkg.is_upgradable)
+def package_split(pkgspec):
+    parts = pkgspec.split('=')
+    if len(parts) > 1:
+        return parts[0], parts[1]
+    else:
+        return parts[0], None
 
-def install(pkgspec, cache, upgrade=False):
-    (installed, upgradable) = package_status(pkgspec, cache)
-    if (not installed) or (upgrade and upgradable):
+def package_status(pkgname, version, cache):
+    try:
+        pkg = cache[pkgname]
+    except KeyError:
+        fail_json(msg="No package matching '%s' is available" % pkgname)
+    if version:
+        return pkg.is_installed and pkg.installed.version == version, False
+    else:
+        return pkg.is_installed, pkg.is_upgradable
+
+def install(pkgspec, cache, upgrade=False, default_release=None):
+    name, version = package_split(pkgspec)
+    installed, upgradable = package_status(name, version, cache)
+    if not installed or (upgrade and upgradable):
         cmd = "%s -q -y install '%s'" % (APT, pkgspec)
+        if default_release:
+            cmd += " -t '%s'" % (default_release,)
         rc, out, err = run_apt(cmd)
         if rc:
-            json_fail(msg="'apt-get install %s' failed: %s" % (pkgspec, err))
+            fail_json(msg="'apt-get install %s' failed: %s" % (pkgspec, err))
         return True
     else:
         return False
 
 def remove(pkgspec, cache, purge=False):
-    (installed, upgradable) = package_status(pkgspec, cache)
+    name, version = package_split(pkgspec)
+    installed, upgradable = package_status(name, version, cache)
     if not installed:
         return False
     else:
         purge = '--purge' if purge else ''
-        cmd = "%s -q -y %s remove '%s'" % (APT, purge, pkgspec)
+        cmd = "%s -q -y %s remove '%s'" % (APT, purge, name)
         rc, out, err = run_apt(cmd)
         if rc:
-            json_fail(msg="'apt-get remove %s' failed: %s" % (pkgspec, err))
+            fail_json(msg="'apt-get remove %s' failed: %s" % (name, err))
         return True
     
 
@@ -111,13 +123,14 @@ if not len(items):
 
 params = {}
 for x in items:
-    (k, v) = x.split("=")
+    (k, v) = x.split("=", 1)
     params[k] = v
 
-state        = params.get('state','installed')
-package      = params.get('pkg', None)
-update_cache = params.get('update-cache', 'no')
-purge        = params.get('purge', 'no')
+state            = params.get('state', 'installed')
+package          = params.get('pkg', params.get('package', params.get('name', None)))
+update_cache     = params.get('update-cache', 'no')
+purge            = params.get('purge', 'no')
+default_release  = params.get('default-release', None)
 
 if state not in ['installed', 'latest', 'removed']:
     fail_json(msg='invalid state')
@@ -132,6 +145,10 @@ if package is None and update_cache != 'yes':
     fail_json(msg='pkg=name and/or update-cache=yes is required')
 
 cache = apt.Cache()
+if default_release:
+    apt_pkg.config['APT::Default-Release'] = default_release
+    # reopen cache w/ modified config
+    cache.open()
 
 if update_cache == 'yes':
     cache.update()
@@ -139,10 +156,16 @@ if update_cache == 'yes':
     if package == None:
         exit_json(changed=False)
 
+if package.count('=') > 1:
+    fail_json(msg='invalid package spec')
+
 if state == 'latest':
-    changed = install(package, cache, upgrade=True)
+    if '=' in package:
+        fail_json(msg='version number inconsistent with state=latest')
+    changed = install(package, cache, upgrade=True,
+                      default_release=default_release)
 elif state == 'installed':
-    changed = install(package, cache)
+    changed = install(package, cache, default_release=default_release)
 elif state == 'removed':
     changed = remove(package, cache, purge == 'yes')
 

library/command

@@ -39,6 +39,26 @@ if args.find("#USE_SHELL") != -1:
    args = args.replace("#USE_SHELL", "")
    shell = True
 
+check_args = shlex.split(args)
+for x in check_args:
+   if x.startswith("creates="):
+       # do not run the command if the line contains creates=filename
+       # and the filename already exists.  This allows idempotence 
+       # of command executions.
+       (k,v) = x.split("=",1)
+       if os.path.exists(v):
+           print json.dumps({
+               "cmd"     : args,
+               "stdout"  : "skipped, since %s exists" % v,
+               "skipped" : True,
+               "changed" : False,
+               "stderr"  : "",
+               "rc"      : 0,
+           })
+           sys.exit(0)
+       args = args.replace(x,'')
+       
+
 if not shell:
     args = shlex.split(args)
 
@@ -71,13 +91,14 @@ if err is None:
    err = ''
 
 result = {
-   "cmd"    : args,
-   "stdout" : out.strip(),
-   "stderr" : err.strip(),
-   "rc"     : cmd.returncode,
-   "start"  : str(startd),
-   "end"    : str(endd),
-   "delta"  : str(delta),
+   "cmd"     : args,
+   "stdout"  : out.strip(),
+   "stderr"  : err.strip(),
+   "rc"      : cmd.returncode,
+   "start"   : str(startd),
+   "end"     : str(endd),
+   "delta"   : str(delta),
+   "changed" : True
 }
 
 print json.dumps(result)

library/copy

@@ -42,7 +42,10 @@ for x in items:
 
 src  = params['src']
 dest = params['dest']
-
+if src:
+   src = os.path.expanduser(src)
+if dest:
+   dest = os.path.expanduser(dest)
  
 # raise an error if there is no src file
 if not os.path.exists(src):

library/fetch

@@ -0,0 +1,24 @@
+#!/usr/bin/python
+
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+### THIS FILE IS FOR REFERENCE OR FUTURE USE ###
+
+# See lib/ansible/runner.py for implementation of the fetch functionality #
+

library/file

@@ -29,6 +29,11 @@ import shutil
 import stat
 import grp
 import pwd
+try:
+    import selinux
+    HAVE_SELINUX=True
+except ImportError:
+    HAVE_SELINUX=False
 
 def debug(msg):
     # ansible ignores stderr, so it's safe to use for debug
@@ -61,10 +66,27 @@ def add_path_info(kwargs):
             kwargs['state'] = 'file'
         else:
             kwargs['state'] = 'directory'
+        if HAVE_SELINUX:
+            kwargs['secontext'] = ':'.join(selinux_context(path))
     else:
         kwargs['state'] = 'absent'
     return kwargs 
  
+# If selinux fails to find a default, return an array of None
+def selinux_default_context(path, mode=0):
+    context = [None, None, None, None]
+    if not HAVE_SELINUX:
+        return context
+    try:
+        ret = selinux.matchpathcon(path, mode)
+    except OSError:
+        return context
+    if ret[0] == -1:
+        return context
+    context = ret[1].split(':')
+    debug("got default secontext=%s" % ret[1])
+    return context
+
 # ===========================================
 
 argfile = sys.argv[1]
@@ -82,7 +104,11 @@ for x in items:
 
 state     = params.get('state','file')
 path      = params.get('path', params.get('dest', params.get('name', None)))
+if path:
+   path = os.path.expanduser(path)
 src       = params.get('src', None)
+if src:
+   src = os.path.expanduser(src)
 dest      = params.get('dest', None)
 mode      = params.get('mode', None)
 owner     = params.get('owner', None)
@@ -91,8 +117,20 @@ group     = params.get('group', None)
 # presently unused, we always use -R (FIXME?)
 recurse   = params.get('recurse', 'false')
 
-# presently unused, implement (FIXME)
-secontext = params.get('secontext', None)
+# selinux related options
+seuser    = params.get('seuser', None)
+serole    = params.get('serole', None)
+setype    = params.get('setype', None)
+selevel   = params.get('serange', 's0')
+context   = params.get('context', None)
+secontext = [seuser, serole, setype, selevel]
+
+if context is not None:
+    if context != 'default':
+        fail_json(msg='invalid context: %s' % context)
+    if seuser is not None or serole is not None or setype is not None:
+        fail_json(msg='cannot define context=default and seuser, serole or setype')
+    secontext = selinux_default_context(path)
 
 if state not in [ 'file', 'directory', 'link', 'absent']:
     fail_json(msg='invalid state: %s' % state)
@@ -119,12 +157,39 @@ def user_and_group(filename):
     debug("got user=%s and group=%s" % (user, group))
     return (user, group)
 
-    
+def selinux_context(path):
+    context = [None, None, None, None]
+    if not HAVE_SELINUX:
+        return context
+    try:
+        ret = selinux.lgetfilecon(path)
+    except:
+        fail_json(path=path, msg='failed to retrieve selinux context')
+    if ret[0] == -1:
+        return context
+    context = ret[1].split(':')
+    debug("got current secontext=%s" % ret[1])
+    return context
+
 def set_context_if_different(path, context, changed):
-   if context is None:
-       return changed
-   if context is not None:
-       fail_json(path=path, msg='context not yet supported')
+    if not HAVE_SELINUX:
+        return changed
+    cur_context = selinux_context(path)
+    new_context = list(cur_context)
+    for i in range(len(context)):
+        if context[i] is not None and context[i] != cur_context[i]:
+            new_context[i] = context[i]
+    debug("current secontext is %s" % ':'.join(cur_context))
+    debug("new secontext is %s" % ':'.join(new_context))
+    if cur_context != new_context:
+        try:
+            rc = selinux.lsetfilecon(path, ':'.join(new_context))
+        except OSError:
+            fail_json(path=path, msg='invalid selinux context')
+        if rc != 0:
+            fail_json(path=path, msg='set selinux context failed')
+        changed = True
+    return changed
     
 def set_owner_if_different(path, owner, changed):
    if owner is None:

library/setup

@@ -19,33 +19,283 @@
 
 DEFAULT_ANSIBLE_SETUP     = "/etc/ansible/setup"
 
+import array
+import fcntl
+import glob
 import sys
 import os
+import platform
+import re
 import shlex
+import socket
+import struct
 import subprocess
+import traceback
 
 try:
     import json
 except ImportError:
     import simplejson as json
 
+_I386RE = re.compile(r'i[3456]86')
+SIOCGIFCONF   = 0x8912
+SIOCGIFHWADDR = 0x8927
+MEMORY_FACTS = ['MemTotal', 'SwapTotal', 'MemFree', 'SwapFree']
+# DMI bits
+DMI_DICT = { 'form_factor':  '/sys/devices/virtual/dmi/id/chassis_type',
+             'product_name': '/sys/devices/virtual/dmi/id/product_name',
+             'product_serial': '/sys/devices/virtual/dmi/id/product_serial',
+             'product_uuid': '/sys/devices/virtual/dmi/id/product_uuid',
+             'product_version': '/sys/devices/virtual/dmi/id/product_version',
+             'system_vendor': '/sys/devices/virtual/dmi/id/sys_vendor' }
+# From smolt and DMI spec
+FORM_FACTOR = [ "Unknown", "Other", "Unknown", "Desktop",
+                "Low Profile Desktop", "Pizza Box", "Mini Tower", "Tower",
+                "Portable", "Laptop", "Notebook", "Hand Held", "Docking Station",
+                "All In One", "Sub Notebook", "Space-saving", "Lunch Box",
+                "Main Server Chassis", "Expansion Chassis", "Sub Chassis",
+                "Bus Expansion Chassis", "Peripheral Chassis", "RAID Chassis",
+                "Rack Mount Chassis", "Sealed-case PC", "Multi-system",
+                "CompactPCI", "AdvancedTCA" ]
+# For the most part, we assume that platform.dist() will tell the truth.
+# This is the fallback to handle unknowns or exceptions
+OSDIST_DICT = { '/etc/redhat-release': 'RedHat',
+                '/etc/vmware-release': 'VMwareESX' }
+
+def get_file_content(path):
+    if os.path.exists(path) and os.access(path, os.R_OK):
+        data = open(path).read().strip()
+        if len(data) == 0:
+            data = None
+    else:
+        data = None
+    return data
+
+# platform.dist() is deprecated in 2.6
+# in 2.6 and newer, you should use platform.linux_distribution()
+def get_distribution_facts(facts):
+    dist = platform.dist()
+    facts['distribution'] = dist[0].capitalize() or 'NA'
+    facts['distribution_version'] = dist[1] or 'NA'
+    facts['distribution_release'] = dist[2] or 'NA'
+    # Try to handle the exceptions now ...
+    for (path, name) in OSDIST_DICT.items():
+        if os.path.exists(path):
+            if facts['distribution'] == 'Fedora':
+                pass
+            elif name == 'RedHat':
+                data = get_file_content(path)
+                if 'Red Hat' in data:
+                    facts['distribution'] = name
+                else:
+                    facts['distribution'] = data.split()[0]
+            else:
+                facts['distribution'] = name
+
+# Platform
+# patform.system() can be Linux, Darwin, Java, or Windows
+def get_platform_facts(facts):
+    facts['system'] = platform.system()
+    facts['kernel'] = platform.release()
+    facts['machine'] = platform.machine()
+    facts['python_version'] = platform.python_version()
+    if facts['machine'] == 'x86_64':
+        facts['architecture'] = facts['machine']
+    elif _I386RE.search(facts['machine']):
+        facts['architecture'] = 'i386'
+    else:
+        facts['archtecture'] = facts['machine']
+    if facts['system'] == 'Linux':
+        get_distribution_facts(facts)
+
+def get_memory_facts(facts):
+    if not os.access("/proc/meminfo", os.R_OK):
+        return facts
+    for line in open("/proc/meminfo").readlines():
+        data = line.split(":", 1)
+        key = data[0]
+        if key in MEMORY_FACTS:
+            val = data[1].strip().split(' ')[0]
+            facts["%s_mb" % key.lower()] = long(val) / 1024
+
+def get_cpu_facts(facts):
+    i = 0
+    physid = 0
+    sockets = {}
+    if not os.access("/proc/cpuinfo", os.R_OK):
+        return facts
+    for line in open("/proc/cpuinfo").readlines():
+        data = line.split(":", 1)
+        key = data[0].strip()
+        if key == 'model name':
+            if 'processor' not in facts:
+                facts['processor'] = []
+            facts['processor'].append(data[1].strip())
+            i += 1
+        elif key == 'physical id':
+            physid = data[1].strip()
+            if physid not in sockets:
+                sockets[physid] = 1
+        elif key == 'cpu cores':
+            sockets[physid] = int(data[1].strip())
+    if len(sockets) > 0:
+        facts['processor_count'] = len(sockets)
+        facts['processor_cores'] = reduce(lambda x, y: x + y, sockets.values())
+    else:
+        facts['processor_count'] = i
+        facts['processor_cores'] = 'NA'
+
+def get_hardware_facts(facts):
+    get_memory_facts(facts)
+    get_cpu_facts(facts)
+    for (key,path) in DMI_DICT.items():
+        data = get_file_content(path)
+        if data is not None:
+            if key == 'form_factor':
+                facts['form_factor'] = FORM_FACTOR[int(data)]
+            else:
+                facts[key] = data
+        else:
+            facts[key] = 'NA'
+
+def get_linux_virtual_facts(facts):
+    if os.path.exists("/proc/xen"):
+        facts['virtualization_type'] = 'xen'
+        facts['virtualization_role'] = 'guest'
+        if os.path.exists("/proc/xen/capabilities"):
+            facts['virtualization_role'] = 'host'
+    if os.path.exists("/proc/modules"):
+        modules = []
+        for line in open("/proc/modules").readlines():
+            data = line.split(" ", 1)
+            modules.append(data[0])
+        if 'kvm' in modules:
+            facts['virtualization_type'] = 'kvm'
+            facts['virtualization_role'] = 'host'
+        elif 'vboxdrv' in modules:
+            facts['virtualization_type'] = 'virtualbox'
+            facts['virtualization_role'] = 'host'
+        elif 'vboxguest' in modules:
+            facts['virtualization_type'] = 'virtualbox'
+            facts['virtualization_role'] = 'guest'
+    if 'QEMU' in facts['processor'][0]:
+        facts['virtualization_type'] = 'kvm'
+        facts['virtualization_role'] = 'guest'
+    if facts['distribution'] == 'VMwareESX':
+        facts['virtualization_type'] = 'VMware'
+        facts['virtualization_role'] = 'host'
+    # You can spawn a dmidecode process and parse that or infer from devices
+    for dev_model in glob.glob('/proc/ide/hd*/model'):
+        info = open(dev_model).read()
+        if 'VMware' in info:
+            facts['virtualization_type'] = 'VMware'
+            facts['virtualization_role'] = 'guest'
+        elif 'Virtual HD' in info or 'Virtual CD' in info:
+            facts['virtualization_type'] = 'VirtualPC'
+            facts['virtualization_role'] = 'guest'
+
+def get_virtual_facts(facts):
+    facts['virtualization_type'] = 'None'
+    facts['virtualization_role'] = 'None'
+    if facts['system'] == 'Linux':
+        facts = get_linux_virtual_facts(facts)
+
+# get list of interfaces that are up
+def get_interfaces():
+    length = 4096
+    offset = 32
+    step = 32
+    if platform.architecture()[0] == '64bit':
+        offset = 16
+        step = 40
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    names = array.array('B', '\0' * length)
+    bytelen = struct.unpack('iL', fcntl.ioctl(
+        s.fileno(), SIOCGIFCONF, struct.pack(
+            'iL', length, names.buffer_info()[0])
+        ))[0]
+    return [names.tostring()[i:i+offset].split('\0', 1)[0]
+            for i in range(0, bytelen, step)]
+
+def get_iface_hwaddr(iface):
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    info = fcntl.ioctl(s.fileno(), SIOCGIFHWADDR,
+                       struct.pack('256s', iface[:15]))
+    return ''.join(['%02x:' % ord(char) for char in info[18:24]])[:-1]
+
+def get_network_facts(facts):
+    facts['fqdn'] = socket.gethostname()
+    facts['hostname'] = facts['fqdn'].split('.')[0]
+    facts['interfaces'] = get_interfaces()
+    for iface in facts['interfaces']:
+        facts[iface] = { 'macaddress': get_iface_hwaddr(iface) }
+        # This is lame, but there doesn't appear to be a good way
+        # to get all addresses for both IPv4 and IPv6.
+        cmd = subprocess.Popen("/sbin/ifconfig %s" % iface, shell=True,
+                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        out, err = cmd.communicate()
+        for line in out.split('\n'):
+            data = line.split()
+            if 'inet addr' in line:
+                if 'ipv4' not in facts[iface]:
+                    facts[iface]['ipv4'] = {}
+                facts[iface]['ipv4'] = { 'address': data[1].split(':')[1],
+                                         'netmask': data[-1].split(':')[1] }
+            if 'inet6 addr' in line:
+                (ip, prefix) = data[2].split('/')
+                scope = data[3].split(':')[1].lower()
+                if 'ipv6' not in facts[iface]:
+                    facts[iface]['ipv6'] = []
+                facts[iface]['ipv6'].append( { 'address': ip,
+                                               'prefix': prefix,
+                                               'scope': scope } )
+    return facts
+
+def get_public_ssh_host_keys(facts):
+    dsa = get_file_content('/etc/ssh/ssh_host_dsa_key.pub')
+    rsa = get_file_content('/etc/ssh/ssh_host_rsa_key.pub')
+    if dsa is None:
+        dsa = 'NA'
+    else:
+        facts['ssh_host_key_dsa_public'] = dsa.split()[1]
+    if rsa is None:
+        rsa = 'NA'
+    else:
+        facts['ssh_host_key_rsa_public'] = rsa.split()[1]
+
+def get_service_facts(facts):
+    get_public_ssh_host_keys(facts)
+
+def ansible_facts():
+    facts = {}
+    get_platform_facts(facts)
+    get_hardware_facts(facts)
+    get_virtual_facts(facts)
+    get_network_facts(facts)
+    get_service_facts(facts)
+    return facts
+
 # load config & template variables
 
 if len(sys.argv) == 1:
     sys.exit(1)
 
+
 argfile = sys.argv[1]
 if not os.path.exists(argfile):
     sys.exit(1)
 
-input_data = shlex.split(open(argfile, 'r').read())
+setup_options = open(argfile).read().strip()
+try:
+   setup_options = json.loads(setup_options)
+except:
+   list_options = shlex.split(setup_options)
+   setup_options = {}
+   for opt in list_options:
+       (k,v) = opt.split("=")
+       setup_options[k]=v   
 
-# turn urlencoded k=v string (space delimited) to regular k=v directionary
-splitted = [x.split('=',1) for x in input_data ]
-splitted = [ (x[0], x[1].replace("~~~"," ")) for x in splitted ]
-new_options = dict(splitted)
-
-ansible_file = new_options.get('metadata', DEFAULT_ANSIBLE_SETUP)
+ansible_file = setup_options.get('metadata', DEFAULT_ANSIBLE_SETUP)
 ansible_dir = os.path.dirname(ansible_file)
 
 # create the config dir if it doesn't exist
@@ -60,6 +310,10 @@ if not os.path.exists(ansible_file):
 else:
     md5sum = os.popen("md5sum %s" % ansible_file).read().split()[0]
 
+# Get some basic facts in case facter or ohai are not installed
+for (k, v) in ansible_facts().items():
+    setup_options["ansible_%s" % k] = v
+
 # if facter is installed, and we can use --json because
 # ruby-json is ALSO installed, include facter data in the JSON
 
@@ -74,7 +328,7 @@ if os.path.exists("/usr/bin/facter"):
        facter = False
    if facter:
        for (k,v) in facter_ds.items():
-           new_options["facter_%s" % k] = v
+           setup_options["facter_%s" % k] = v
 
 # ditto for ohai, but just top level string keys
 # because it contains a lot of nested stuff we can't use for
@@ -93,13 +347,13 @@ if os.path.exists("/usr/bin/ohai"):
        for (k,v) in ohai_ds.items():
            if type(v) == str or type(v) == unicode:
                k2 = "ohai_%s" % k
-               new_options[k2] = v
+               setup_options[k2] = v
 
 # write the template/settings file using
 # instructions from server
 
 f = open(ansible_file, "w+")
-reformat = json.dumps(new_options, sort_keys=True, indent=4)
+reformat = json.dumps(setup_options, sort_keys=True, indent=4)
 f.write(reformat)
 f.close()
 
@@ -108,9 +362,9 @@ md5sum2 = os.popen("md5sum %s" % ansible_file).read().split()[0]
 if md5sum != md5sum2:
    changed = True
 
-new_options['written'] = ansible_file
-new_options['changed'] = changed
-new_options['md5sum']  = md5sum2 
+setup_options['written'] = ansible_file
+setup_options['changed'] = changed
+setup_options['md5sum']  = md5sum2 
 
-print json.dumps(new_options)
+print json.dumps(setup_options)
 

library/slurp

@@ -0,0 +1,71 @@
+#!/usr/bin/python
+
+# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import shlex
+import base64
+
+try:
+   import json
+except ImportError:
+   import simplejson as json
+
+# ===========================================
+# convert arguments of form a=b c=d
+# to a dictionary
+
+if len(sys.argv) == 1:
+    sys.exit(1)
+argfile = sys.argv[1]
+if not os.path.exists(argfile):
+    sys.exit(1)
+items = shlex.split(open(argfile, 'r').read())
+
+params = {}
+for x in items:
+    (k, v) = x.split("=")
+    params[k] = v
+source  = os.path.expanduser(params['src'])
+
+# ==========================================
+ 
+# raise an error if there is no template metadata
+if not os.path.exists(source):
+    print json.dumps(dict(
+        failed = 1,
+        msg    = "file not found: %s" % source
+    ))
+    sys.exit(1)
+
+if not os.access(source, os.R_OK):
+    print json.dumps(dict(
+        failed = 1,
+        msg    = "file is not readable: %s" % source
+    ))
+    sys.exit(1)
+
+# ==========================================
+
+data = file(source).read()
+data = base64.b64encode(data)
+
+print json.dumps(dict(content=data, encoding='base64'))
+sys.exit(0)
+

library/template

@@ -17,106 +17,8 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-import sys
-import os
-import jinja2
-import shlex
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-environment = jinja2.Environment()
-
-# ===========================================
-# convert arguments of form a=b c=d
-# to a dictionary
-# FIXME: make more idiomatic
-
-if len(sys.argv) == 1:
-    sys.exit(1)
-argfile = sys.argv[1]
-if not os.path.exists(argfile):
-    sys.exit(1)
-items = shlex.split(open(argfile, 'r').read())
-
-params = {}
-for x in items:
-    (k, v) = x.split("=")
-    params[k] = v
-
-source  = params['src']
-dest    = params['dest']
-metadata = params.get('metadata', '/etc/ansible/setup') 
-
- 
-# raise an error if there is no template metadata
-if not os.path.exists(metadata):
-    print json.dumps({
-        "failed" : 1,
-        "msg"    : "Missing %s, did you run the setup module yet?" % metadata
-    })
-    sys.exit(1)
-
-# raise an error if we can't parse the template metadata
-#data = {}
-try:
-   f = open(metadata)
-   data = json.loads(f.read())
-   f.close()
-except:
-   print json.dumps({
-       "failed" : 1,
-       "msg"    : "Failed to parse/load %s, rerun the setup module?" % metadata
-   })
-   sys.exit(1)
-
-if not os.path.exists(source):
-    print json.dumps({
-        "failed" : 1,
-        "msg"    : "Source template could not be read: %s" % source
-    })
-    sys.exit(1)
-
-source = file(source).read()
-
-if os.path.isdir(dest):
-    print json.dumps({
-         "failed" : 1,
-         "msg"    : "Destination is a directory"
-    })
-    sys.exit(1)
-
-# record md5sum of original source file so we can report if it changed
-changed = False
-md5sum = None
-if os.path.exists(dest):
-    md5sum = os.popen("md5sum %s" % dest).read().split()[0]
-
-try:
-    # call Jinja2 here and save the new template file
-    template = environment.from_string(source)
-    data_out = template.render(data)
-except jinja2.TemplateError, e:
-    print json.dumps({
-        "failed": True,
-        "msg"   : e.message
-    })
-    sys.exit(1)
-f = open(dest, "w+")
-f.write(data_out)
-f.close()
-
-# record m5sum and return success and whether things have changed
-md5sum2 = os.popen("md5sum %s" % dest).read().split()[0]
-
-if md5sum != md5sum2:
-    changed = True
-
-# mission accomplished
-print json.dumps({
-   "md5sum"   : md5sum2,
-   "changed"  : changed 
-})
-
+# hey the Ansible template module isn't really a remote transferred
+# module.  All the magic happens in Runner.py making use of the
+# copy module, and if not running from a playbook, also the 'slurp'
+# module.
 

library/virt

@@ -0,0 +1,450 @@
+#!/usr/bin/python 
+"""
+Virt management features
+
+Copyright 2007, 2012 Red Hat, Inc
+Michael DeHaan <mdehaan@redhat.com>
+Seth Vidal <skvidal@fedoraproject.org>
+
+This software may be freely redistributed under the terms of the GNU
+general public license.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+VIRT_FAILED = 1
+VIRT_SUCCESS = 0
+VIRT_UNAVAILABLE=2
+
+try:
+    import json
+except ImportError:
+    import simplejson as json
+
+
+# other modules
+import os
+import sys
+import subprocess
+try:
+    import libvirt
+except ImportError:
+    print json.dumps({
+        "failed" : True,
+        "msg" : "libvirt python module unavailable",
+        "rc": VIRT_UNAVAILABLE,
+        })   
+    sys.exit(1)
+
+
+import shlex
+
+VIRT_STATE_NAME_MAP = {
+   0 : "running",
+   1 : "running",
+   2 : "running",
+   3 : "paused",
+   4 : "shutdown",
+   5 : "shutdown",
+   6 : "crashed"
+}
+
+class LibvirtConnection(object):
+
+    def __init__(self):
+
+        cmd = subprocess.Popen("uname -r", shell=True, stdout=subprocess.PIPE,
+                                close_fds=True)
+        output = cmd.communicate()[0]
+
+        if output.find("xen") != -1:
+            conn = libvirt.open(None)
+        else:
+            conn = libvirt.open("qemu:///system")
+
+        if not conn:
+            raise Exception("hypervisor connection failure")
+
+        self.conn = conn
+
+    def find_vm(self, vmid):
+        """
+        Extra bonus feature: vmid = -1 returns a list of everything
+        """
+        conn = self.conn
+
+        vms = []
+
+        # this block of code borrowed from virt-manager:
+        # get working domain's name
+        ids = conn.listDomainsID()
+        for id in ids:
+            vm = conn.lookupByID(id)
+            vms.append(vm)
+        # get defined domain
+        names = conn.listDefinedDomains()
+        for name in names:
+            vm = conn.lookupByName(name)
+            vms.append(vm)
+
+        if vmid == -1:
+            return vms
+
+        for vm in vms:
+            if vm.name() == vmid:
+                return vm
+
+        raise Exception("virtual machine %s not found" % vmid)
+
+    def shutdown(self, vmid):
+        return self.find_vm(vmid).shutdown()
+
+    def pause(self, vmid):
+        return self.suspend(self.conn,vmid)
+
+    def unpause(self, vmid):
+        return self.resume(self.conn,vmid)
+
+    def suspend(self, vmid):
+        return self.find_vm(vmid).suspend()
+
+    def resume(self, vmid):
+        return self.find_vm(vmid).resume()
+
+    def create(self, vmid):
+        return self.find_vm(vmid).create()
+
+    def destroy(self, vmid):
+        return self.find_vm(vmid).destroy()
+
+    def undefine(self, vmid):
+        return self.find_vm(vmid).undefine()
+
+    def get_status2(self, vm):
+        state = vm.info()[0]
+        # print "DEBUG: state: %s" % state
+        return VIRT_STATE_NAME_MAP.get(state,"unknown")
+
+    def get_status(self, vmid):
+        state = self.find_vm(vmid).info()[0]
+        return VIRT_STATE_NAME_MAP.get(state,"unknown")
+
+    def nodeinfo(self):
+        return self.conn.getInfo()
+
+    def get_type(self):
+        return self.conn.getType()
+
+    def get_maxVcpus(self, vmid):
+        vm = self.conn.lookupByName(vmid)
+        return vm.maxVcpus()
+
+    def get_maxMemory(self, vmid):
+        vm = self.conn.lookupByName(vmid)
+        return vm.maxMemory()
+
+    def getFreeMemory(self):
+        return self.conn.getFreeMemory()
+
+    def get_autostart(self, vmid):
+        vm = self.conn.lookupByName(vmid)
+        return vm.autostart()
+    
+    def set_autostart(self, vmid, val):
+        vm = self.conn.lookupByName(vmid)
+        return vm.setAutostart(val)
+    
+
+
+class Virt(object):
+
+    def __get_conn(self):
+        self.conn = LibvirtConnection()
+        return self.conn
+
+    def get_vm(self, vmid):
+        self.__get_conn()
+        return self.conn.find_vm(vmid)
+        
+    def state(self):
+        vms = self.list_vms()
+        state = []
+        for vm in vms:
+            state_blurb = self.conn.get_status(vm)
+            state.append("%s %s" % (vm,state_blurb))
+        return state
+
+
+    def info(self):
+        vms = self.list_vms()
+        info = dict()
+        for vm in vms:
+            data = self.conn.find_vm(vm).info()
+            # libvirt returns maxMem, memory, and cpuTime as long()'s, which
+            # xmlrpclib tries to convert to regular int's during serialization.
+            # This throws exceptions, so convert them to strings here and
+            # assume the other end of the xmlrpc connection can figure things
+            # out or doesn't care.
+            info[vm] = {
+                "state"     : VIRT_STATE_NAME_MAP.get(data[0],"unknown"),
+                "maxMem"    : str(data[1]),
+                "memory"    : str(data[2]),
+                "nrVirtCpu" : data[3],
+                "cpuTime"   : str(data[4]),
+            }
+            info[vm]["autostart"] =  self.conn.get_autostart(vm)
+
+        return info
+
+    def nodeinfo(self):
+        self.__get_conn()
+        info = dict()
+        data = self.conn.nodeinfo()
+        info = {
+            "cpumodel"     : str(data[0]),
+            "phymemory"    : str(data[1]),
+            "cpus"         : str(data[2]),
+            "cpumhz"       : str(data[3]),
+            "numanodes"    : str(data[4]),
+            "sockets"      : str(data[5]),
+            "cpucores"     : str(data[6]),
+            "cputhreads"   : str(data[7])
+        }
+        return info
+
+    def list_vms(self):
+        self.conn = self.__get_conn()
+        vms = self.conn.find_vm(-1)
+        results = []
+        for x in vms:
+            try:
+                results.append(x.name())
+            except:
+                pass
+        return results
+
+    def virttype(self):
+        return self.__get_conn().get_type()
+        
+
+    def autostart(self, vmid):
+        self.conn = self.__get_conn()
+        return self.conn.set_autostart(vmid, True)
+
+    def freemem(self):
+        self.conn = self.__get_conn()
+        return self.conn.getFreeMemory()
+
+    def shutdown(self, vmid):
+        """
+        Make the machine with the given vmid stop running.
+        Whatever that takes.
+        """
+        self.__get_conn()
+        self.conn.shutdown(vmid)
+        return 0
+
+
+    def pause(self, vmid):
+
+        """
+        Pause the machine with the given vmid.
+        """
+        self.__get_conn()
+        return self.conn.suspend(vmid)
+
+
+    def unpause(self, vmid):
+
+        """
+        Unpause the machine with the given vmid.
+        """
+
+        self.__get_conn()
+        return self.conn.resume(vmid)
+
+
+    def create(self, vmid):
+
+        """
+        Start the machine via the given mac address.
+        """
+        self.__get_conn()
+        return self.conn.create(vmid)
+
+    def start(self, vmid):
+
+        """
+        Start the machine via the given id/name
+        """
+        self.__get_conn()
+        return self.conn.create(vmid)
+
+
+    def destroy(self, vmid):
+
+        """
+        Pull the virtual power from the virtual domain, giving it virtually no
+        time to virtually shut down.
+        """
+        self.__get_conn()
+        return self.conn.destroy(vmid)
+
+
+    def undefine(self, vmid):
+
+        """
+        Stop a domain, and then wipe it from the face of the earth.
+        by deleting the disk image and its configuration file.
+        """
+
+        self.__get_conn()
+        return self.conn.undefine(vmid)
+
+
+    def status(self, vmid):
+
+        """
+        Return a state suitable for server consumption.  Aka, codes.py values, not XM output.
+        """
+
+        self.__get_conn()
+        return self.conn.get_status(vmid)
+
+
+    def get_xml(self, vmid):
+        """
+        Receive a Vm id as input
+        Return an xml describing vm config returned by a libvirt call
+        """
+        conn = libvirt.openReadOnly(None)
+        if conn == None:
+            return (-1,'Failed to open connection to the hypervisor')
+        try:
+            domV = conn.lookupByName(vmid)
+        except:
+            return (-1,'Failed to find the main domain')
+        return domV.XMLDesc(0)
+
+
+    def get_maxVcpus(self, vmid):
+        """
+        Gets the max number of VCPUs on a guest
+        """
+
+        self.__get_conn()
+        return self.conn.get_maxVcpus(vmid)
+
+    def get_max_memory(self, vmid):
+        """
+        Gets the max memory on a guest
+        """
+
+        self.__get_conn()
+        return self.conn.get_MaxMemory(vmid)
+
+def main():
+    rc = VIRT_SUCCESS
+    vm_commands = ['create','status', 'start', 'stop', 'pause', 'unpause', 
+                'shutdown', 'undefine', 'destroy', 'get_xml', 'autostart']
+    host_commands = ['freemem', 'list_vms', 'info', 'nodeinfo', 'virttype']
+    
+    msg = """
+    virtmodule arguments:
+      state=[running|shutdown] guest=guestname
+      command=some_virt_command  [guest=guestname]
+    """
+    
+    if len(sys.argv) == 1:
+        return VIRT_FAILED, msg
+
+    argfile = sys.argv[1]
+    if not os.path.exists(argfile):
+        msg = "Argument file not found"
+        return VIRT_FAILED, msg
+
+    args = open(argfile, 'r').read()
+    items = shlex.split(args)
+
+    if not len(items):
+        return VIRT_FAILED, msg
+
+
+    # guest=name state=[running|shutdown|destroyed|undefined]
+    # command=[some command] [guest=name]
+    
+    params = {}
+    if '=' not in args:
+        msg = "No proper arguments provided to virt module: %s" % args
+        return VIRT_FAILED, msg
+        
+    for x in items:
+        (k, v) = x.split("=")
+        params[k] = v
+
+    state      = params.get('state', None)
+    guest      = params.get('guest', None)
+    command    = params.get('command', None)
+    options    = params.get('options', [])
+
+    v = Virt()
+    res = {}
+    
+    if state:
+        if not guest:
+            msg = "state change requires a guest specified"
+            return VIRT_FAILED, msg
+      
+        res['changed'] = False
+        if state == 'running':
+            if v.status(guest) is not 'running':
+                res['changed'] = True 
+                res['msg'] = v.start(guest)
+        elif state == 'shutdown':
+            if v.status(guest) is not 'shutdown':
+                res['changed'] = True
+                res['msg'] = v.shutdown(guest)
+        
+        return VIRT_SUCCESS, res
+        
+    if command:
+        if command in vm_commands:
+            if not guest:
+                msg = "%s requires 1 argument: guest" % command
+                return VIRT_FAILED, msg
+                             
+            res = getattr(v, command)(guest)
+            if type(res) != dict:
+                res = { command: res }
+            return rc, res
+        
+        elif hasattr(v, command):
+            res = getattr(v, command)()
+            if type(res) != dict:
+                res = { command: res }
+            return rc, res
+            
+        else:
+            msg = "Command %s not recognized" % basecmd
+            rc = VIRT_FAILED
+        
+    return rc, msg
+    
+if __name__ == "__main__":
+    try:
+        rc, result = main()
+    except Exception, e:
+        rc = 1
+        result = str(e)
+
+    if rc != 0: # something went wrong emit the msg
+        print json.dumps({
+            "failed" : rc,
+            "msg" : result,
+            "rc": rc,
+        })   
+        sys.exit(rc)
+    else:
+        print json.dumps(result)
+    

library/yum

@@ -68,7 +68,7 @@ def pkg_to_dict(po):
     return d
     
 def list_stuff(my, stuff):
-    # FIXME - there are poitential tracebacks that could occur here
+    # FIXME - there are potential tracebacks that could occur here
     # need some more catching for them so we can see what happened
     if stuff == 'installed':
         return [ pkg_to_dict(po) for po in my.rpmdb ]
@@ -317,22 +317,18 @@ def main():
         except Exception, e:
             return 1, str(e)
             
-    elif 'state' in params:
-        if 'pkg' not in params:
+    else:
+        pkg = params.get('pkg', params.get('package', params.get('name', None))) 
+        if 'pkg' is None:
             results['msg'] = "No pkg specified"
         else:
             try:
                 my = yum_base(conf_file=params['conf_file'], cachedir=True)        
-                state = params['state']
-                pkgspec = params['pkg']
-                results = ensure(my, state, pkgspec)
+                state = params.get('state', 'installed')
+                results = ensure(my, state, pkg)
             except Exception, e:
                 return 1, str(e)
 
-    else:
-        print json.dumps(dict(failed=True, msg='invalid module parameters'))
-        sys.exit(1)
-
     print json.dumps(results)
     return 0, None
 

packaging/arch/PKGBUILD

@@ -0,0 +1,44 @@
+#Maintainer: Michel Blanc <mblanc@erasme.org>
+pkgname=ansible-git
+pkgver=20120419
+pkgrel=1
+pkgdesc="A radically simple deployment, model-driven configuration management, and command execution framework"
+arch=('any')
+url="https://github.com/ansible/ansible"
+license=('GPL3')
+depends=('python2' 'python2-yaml' 'python-paramiko>=1.7.7' 'python2-jinja' 'python-simplejson')
+makedepends=('git' 'asciidoc' 'fakeroot')
+
+_gitroot="https://github.com/ansible/ansible"
+_gitname="ansible"
+
+build() {
+  cd "$srcdir"
+  msg "Connecting to GIT server...."
+
+  if [ -d $_gitname ] ; then
+    cd $_gitname && git pull origin
+    msg "The local files are updated."
+  else
+    git clone $_gitroot $_gitname
+  fi
+
+  msg "GIT checkout done or server timeout"
+
+  cd "$srcdir/$_gitname"
+  make
+}
+
+package() {
+  cd "$srcdir/$_gitname"
+
+  mkdir -p ${pkgdir}/usr/share/ansible
+  cp ./library/* ${pkgdir}/usr/share/ansible/
+  python setup.py install -O1 --root=${pkgdir}
+
+  install -D docs/man/man1/ansible.1 ${pkgdir}/usr/share/man/man1/ansible.1
+  install -D docs/man/man1/ansible-playbook.1 ${pkgdir}/usr/share/man/man1/ansible-playbook.1
+
+  gzip -9 ${pkgdir}/usr/share/man/man1/ansible.1
+  gzip -9 ${pkgdir}/usr/share/man/man1/ansible-playbook.1
+}

packaging/debian/README.txt

@@ -0,0 +1,11 @@
+I have added a debian folder for use in building a .deb file for ansible. From the ansible directory you can run the following command to construct a debian package of ansible.
+
+~/ansible$ dpkg-buildpackage -us -uc -rfakeroot
+
+The debian package files will be placed in the ../ directory and can be installed with the following command:
+~/$ sudo dpkg -i .deb
+
+Dpkg -i doesn't resolve dependencies, so if the previous command fails because of dependencies, you will need to run the following to install the dependencies (if needed) and then re-run the dpkg -i command to install the package:
+$ sudo apt-get -f install
+
+--Henry Graham

packaging/debian/ansible.dirs

@@ -0,0 +1,3 @@
+etc/ansible
+usr/lib/python2.7/site-packages
+usr/share/ansible

packaging/debian/ansible.install

@@ -0,0 +1,5 @@
+examples/hosts etc/ansible
+library/* usr/share/ansible
+docs/man/man1/ansible.1 usr/share/man/man1
+docs/man/man1/ansible-playbook.1 usr/share/man/man1
+bin/* usr/bin

packaging/debian/changelog

@@ -0,0 +1,11 @@
+ansible (0.3) debian; urgency=low
+
+  * 0.3 update
+
+ -- Michael DeHaan <michael.dehaan@gmail.com> Mon, 23 Apr 2012 11:08:00 -0400
+
+ansible (0.0.2) debian; urgency=low
+
+  * Initial Release
+
+ -- Henry Graham (hzgraham) <Henry.Graham@mail.wvu.edu>  Tue, 17 Apr 2012 17:17:01 -0400

packaging/debian/compat

@@ -0,0 +1 @@
+5

packaging/debian/control

@@ -0,0 +1,13 @@
+Source: ansible
+Section: admin
+Priority: optional
+Maintainer: Henry Graham (hzgraham) <Henry.Graham@mail.wvu.edu>
+Build-Depends: cdbs, debhelper (>= 5.0.0)
+Standards-Version: 3.9.1
+Homepage: http://ansible.github.com/
+
+Package: ansible
+Architecture: all
+Depends: python, python-support (>= 0.90), python-jinja2, python-yaml, python-paramiko
+Description: Ansible Application
+ Ansible is a extra-simple tool/API for doing 'parallel remote things' over SSH executing commands, running "modules", or executing larger 'playbooks' that can serve as a configuration management or deployment system.

packaging/debian/copyright

@@ -0,0 +1,26 @@
+This package was debianized by Henry Graham (hzgraham) <Henry.Graham@mail.wvu.edu> on
+Tue, 17 Apr 2012 12:19:47 -0400.
+
+It was downloaded from https://github.com/ansible/ansible.git
+
+Copyright: Henry Graham (hzgraham) <Henry.Graham@mail.wvu.edu>
+
+License:
+
+   This package is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; version 2 dated June, 1991.
+
+   This package is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this package; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301,
+   USA.
+
+On Debian systems, the complete text of the GNU General
+Public License can be found in `/usr/share/common-licenses/GPL'.
+

packaging/debian/docs

@@ -0,0 +1 @@
+README.md

packaging/debian/pycompat

@@ -0,0 +1 @@
+2

packaging/debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+# -- makefile --
+
+include /usr/share/cdbs/1/rules/debhelper.mk
+DEB_PYTHON_SYSTEM = pysupport
+include /usr/share/cdbs/1/class/python-distutils.mk

packaging/gentoo/README.md

@@ -0,0 +1,3 @@
+Gentoo ebuilds are available here:
+
+https://github.com/uu/ubuilds

packaging/rpm/ansible.spec

@@ -1,40 +1,45 @@
+%if 0%{?rhel} <= 5
 %{!?python_sitelib: %global python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")}
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
+%endif
 
 Name: ansible
-Release: 1
+Release: 1%{?dist}
 Summary: Minimal SSH command and control
-Version: 0.0.2
+Version: 0.3
 
-BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Group: Development/Libraries
 License: GPLv3
-Prefix: %{_prefix}
 Source0: https://github.com/downloads/ansible/ansible/%{name}-%{version}.tar.gz
 Url: http://ansible.github.com
 
 BuildArch: noarch
-BuildRequires: asciidoc
-BuildRequires: python-devel
+BuildRequires: python2-devel
 
+Requires: PyYAML
 Requires: python-paramiko
 Requires: python-jinja2
 
 %description
-Ansible is a extra-simple tool/API for doing 'parallel remote things' over SSH
-executing commands, running "modules", or executing larger 'playbooks' that
-can serve as a configuration management or deployment system.
+
+Ansible is a radically simple model-driven configuration management,
+multi-node deployment, and remote task execution system. Ansible works
+over SSH and does not require any software or daemons to be installed
+on remote nodes. Extension modules can be written in any language and
+are transferred to managed machines automatically.
+
 
 %prep
-%setup -q -n %{name}-%{version}
+%setup -q
 
 %build
-python setup.py build
+%{__python} setup.py build
 
 %install
-python setup.py install -O1 --root=$RPM_BUILD_ROOT --record=INSTALLED_FILES
+%{__python} setup.py install -O1 --root=$RPM_BUILD_ROOT
 mkdir -p $RPM_BUILD_ROOT/etc/ansible/
 cp examples/hosts $RPM_BUILD_ROOT/etc/ansible/
-mkdir -p $RPM_BUILD_ROOT/%{_mandir}/man1/ 
+mkdir -p $RPM_BUILD_ROOT/%{_mandir}/man1/
 cp -v docs/man/man1/*.1 $RPM_BUILD_ROOT/%{_mandir}/man1/
 mkdir -p $RPM_BUILD_ROOT/%{_datadir}/ansible
 cp -v library/* $RPM_BUILD_ROOT/%{_datadir}/ansible/
@@ -43,17 +48,19 @@ cp -v library/* $RPM_BUILD_ROOT/%{_datadir}/ansible/
 rm -rf $RPM_BUILD_ROOT
 
 %files
-%doc README.md PKG-INFO
 %defattr(-,root,root)
-%{_mandir}/man1/*.gz
-%{python_sitelib}/*
+%{python_sitelib}/ansible*
 %{_bindir}/ansible*
-%{_datadir}/ansible/*
-%config(noreplace) /etc/ansible/hosts
-%config(noreplace) %{_sysconfdir}/ansible/
+%{_datadir}/ansible
+%config(noreplace) %{_sysconfdir}/ansible
+%doc README.md PKG-INFO
+%doc %{_mandir}/man1/ansible*
 
 
 %changelog
+* Mon Apr 23 2012 Michael DeHaan <mdehaan@redhat.com> - 0.3-1
+- Release of 0.3
+
 * Tue Apr  3 2012 John Eckersberg <jeckersb@redhat.com> - 0.0.2-1
 - Release of 0.0.2
 

setup.py

@@ -4,7 +4,6 @@
 # for you, you should go through "make install" or "make RPMs" 
 # for that, or manually copy modules over.
 
-import glob
 import os
 import sys
 
@@ -19,7 +18,8 @@ setup(name='ansible',
       author_email='michael.dehaan@gmail.com',
       url='http://ansible.github.com/',
       license='GPLv3',
-      package_dir = { 'ansible' : 'lib/ansible' },
+      install_requires=['paramiko', 'jinja2', "PyYAML"],
+      package_dir={ 'ansible': 'lib/ansible' },
       packages=[
          'ansible',
       ],

test/TestInventory.py

@@ -0,0 +1,252 @@
+import os
+import unittest
+
+from ansible.inventory import Inventory
+from ansible.runner import Runner
+
+class TestInventory(unittest.TestCase):
+
+    def setUp(self):
+        self.cwd = os.getcwd()
+        self.test_dir = os.path.join(self.cwd, 'test')
+
+        self.inventory_file = os.path.join(self.test_dir, 'simple_hosts')
+        self.inventory_script = os.path.join(self.test_dir, 'inventory_api.py')
+        self.inventory_yaml = os.path.join(self.test_dir, 'yaml_hosts')
+
+        os.chmod(self.inventory_script, 0755)
+
+    def tearDown(self):
+        os.chmod(self.inventory_script, 0644)
+
+    ### Simple inventory format tests
+
+    def simple_inventory(self):
+        return Inventory( self.inventory_file )
+
+    def script_inventory(self):
+        return Inventory( self.inventory_script )
+
+    def yaml_inventory(self):
+        return Inventory( self.inventory_yaml )
+
+    def test_simple(self):
+        inventory = self.simple_inventory()
+        hosts = inventory.list_hosts()
+
+        expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+    def test_simple_all(self):
+        inventory = self.simple_inventory()
+        hosts = inventory.list_hosts('all')
+
+        expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+    def test_simple_norse(self):
+        inventory = self.simple_inventory()
+        hosts = inventory.list_hosts("norse")
+
+        expected_hosts=['thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+    def test_simple_ungrouped(self):
+        inventory = self.simple_inventory()
+        hosts = inventory.list_hosts("ungrouped")
+
+        expected_hosts=['jupiter', 'saturn']
+        assert hosts == expected_hosts
+
+    def test_simple_combined(self):
+        inventory = self.simple_inventory()
+        hosts = inventory.list_hosts("norse:greek")
+
+        expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+    def test_simple_restrict(self):
+        inventory = self.simple_inventory()
+
+        restricted_hosts = ['hera', 'poseidon', 'thor']
+        expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+
+        inventory.restrict_to(restricted_hosts)
+        hosts = inventory.list_hosts("norse:greek")
+
+        assert hosts == restricted_hosts
+
+        inventory.lift_restriction()
+        hosts = inventory.list_hosts("norse:greek")
+
+        assert hosts == expected_hosts
+
+    def test_simple_vars(self):
+        inventory = self.simple_inventory()
+        vars = inventory.get_variables('thor')
+
+        assert vars == {}
+
+    def test_simple_port(self):
+        inventory = self.simple_inventory()
+        vars = inventory.get_variables('hera')
+
+        assert vars == {'ansible_ssh_port': 3000}
+
+    ### Inventory API tests
+
+    def test_script(self):
+        inventory = self.script_inventory()
+        hosts = inventory.list_hosts()
+
+        expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+
+        print "Expected: %s"%(expected_hosts)
+        print "Got     : %s"%(hosts)
+        assert sorted(hosts) == sorted(expected_hosts)
+
+    def test_script_all(self):
+        inventory = self.script_inventory()
+        hosts = inventory.list_hosts('all')
+
+        expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert sorted(hosts) == sorted(expected_hosts)
+
+    def test_script_norse(self):
+        inventory = self.script_inventory()
+        hosts = inventory.list_hosts("norse")
+
+        expected_hosts=['thor', 'odin', 'loki']
+        assert sorted(hosts) == sorted(expected_hosts)
+
+    def test_script_combined(self):
+        inventory = self.script_inventory()
+        hosts = inventory.list_hosts("norse:greek")
+
+        expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert sorted(hosts) == sorted(expected_hosts)
+
+    def test_script_restrict(self):
+        inventory = self.script_inventory()
+
+        restricted_hosts = ['hera', 'poseidon', 'thor']
+        expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+
+        inventory.restrict_to(restricted_hosts)
+        hosts = inventory.list_hosts("norse:greek")
+
+        assert sorted(hosts) == sorted(restricted_hosts)
+
+        inventory.lift_restriction()
+        hosts = inventory.list_hosts("norse:greek")
+
+        assert sorted(hosts) == sorted(expected_hosts)
+
+    def test_script_vars(self):
+        inventory = self.script_inventory()
+        vars = inventory.get_variables('thor')
+
+        assert vars == {"hammer":True}
+
+    ### Tests for yaml inventory file
+
+    def test_yaml(self):
+        inventory = self.yaml_inventory()
+        hosts = inventory.list_hosts()
+        print hosts
+        expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+    def test_yaml_all(self):
+        inventory = self.yaml_inventory()
+        hosts = inventory.list_hosts('all')
+
+        expected_hosts=['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+    def test_yaml_norse(self):
+        inventory = self.yaml_inventory()
+        hosts = inventory.list_hosts("norse")
+
+        expected_hosts=['thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+    def test_simple_ungrouped(self):
+        inventory = self.yaml_inventory()
+        hosts = inventory.list_hosts("ungrouped")
+
+        expected_hosts=['jupiter']
+        assert hosts == expected_hosts
+
+    def test_yaml_combined(self):
+        inventory = self.yaml_inventory()
+        hosts = inventory.list_hosts("norse:greek")
+
+        expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+    def test_yaml_restrict(self):
+        inventory = self.yaml_inventory()
+
+        restricted_hosts = ['hera', 'poseidon', 'thor']
+        expected_hosts=['zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+
+        inventory.restrict_to(restricted_hosts)
+        hosts = inventory.list_hosts("norse:greek")
+
+        assert hosts == restricted_hosts
+
+        inventory.lift_restriction()
+        hosts = inventory.list_hosts("norse:greek")
+
+        assert hosts == expected_hosts
+
+    def test_yaml_vars(self):
+        inventory = self.yaml_inventory()
+        vars = inventory.get_variables('thor')
+
+        assert vars == {"hammer":True}
+
+    def test_yaml_change_vars(self):
+        inventory = self.yaml_inventory()
+        vars = inventory.get_variables('thor')
+
+        vars["hammer"] = False
+
+        vars = inventory.get_variables('thor')
+        assert vars == {"hammer":True}
+
+    def test_yaml_host_vars(self):
+        inventory = self.yaml_inventory()
+        vars = inventory.get_variables('saturn')
+
+        assert vars == {"moon":"titan", "moon2":"enceladus"}
+
+    def test_yaml_port(self):
+        inventory = self.yaml_inventory()
+        vars = inventory.get_variables('hera')
+
+        assert vars == {'ansible_ssh_port': 3000, 'ntp_server': 'olympus.example.com'}
+
+    ### Test Runner class method
+
+    def test_class_method(self):
+        hosts, groups = Runner.parse_hosts(self.inventory_file)
+
+        expected_hosts = ['jupiter', 'saturn', 'zeus', 'hera', 'poseidon', 'thor', 'odin', 'loki']
+        assert hosts == expected_hosts
+
+        expected_groups= {
+            'ungrouped': ['jupiter', 'saturn'],
+            'greek': ['zeus', 'hera', 'poseidon'],
+            'norse': ['thor', 'odin', 'loki']
+        }
+        assert groups == expected_groups
+
+    def test_class_override(self):
+        override_hosts = ['thor', 'odin']
+        hosts, groups = Runner.parse_hosts(self.inventory_file, override_hosts)
+
+        assert hosts == override_hosts
+
+        assert groups == { 'ungrouped': override_hosts }

test/TestPlayBook.py

@@ -89,8 +89,10 @@ class TestCallbacks(object):
     def on_setup_secondary(self):
         pass
 
+    def on_no_hosts(self):
+        pass
 
-class TestRunner(unittest.TestCase):
+class TestPlaybook(unittest.TestCase):
 
    def setUp(self):
        self.user = getpass.getuser()
@@ -134,25 +136,33 @@ class TestRunner(unittest.TestCase):
            timeout      = 5,
            remote_user  = self.user,
            remote_pass  = None,
-           verbose      = False,
            stats            = ans_callbacks.AggregateStats(),
            callbacks        = self.test_callbacks,
            runner_callbacks = self.test_callbacks
        )
-       results = self.playbook.run()
-       return dict(
-           results = results,
-           events = EVENTS
-       ) 
+       result = self.playbook.run()
+       print utils.bigjson(dict(events=EVENTS))
+       return result
 
    def test_one(self):
        pb = os.path.join(self.test_dir, 'playbook1.yml')
-       expected = os.path.join(self.test_dir, 'playbook1.events')
-       expected = utils.json_loads(file(expected).read())
        actual = self._run(pb)
+
        # if different, this will output to screen 
+       print "**ACTUAL**"
        print utils.bigjson(actual)
-       assert cmp(expected, actual) == 0, "expected events match actual events"
+       expected =  { 
+            "127.0.0.2": {
+                "changed": 9, 
+                "failures": 0, 
+                "ok": 12, 
+                "skipped": 1, 
+                "unreachable": 0
+            }
+       }
+       print "**EXPECTED**"
+       print utils.bigjson(expected)
+       assert utils.bigjson(expected) == utils.bigjson(actual)
 
        # make sure the template module took options from the vars section
        data = file('/tmp/ansible_test_data_template.out').read()

test/TestRunner.py

@@ -14,6 +14,15 @@ try:
 except:
    import simplejson as json
 
+from nose.plugins.skip import SkipTest
+
+def get_binary(name):
+    for directory in os.environ["PATH"].split(os.pathsep):
+        path = os.path.join(directory, name)
+        if os.path.isfile(path) and os.access(path, os.X_OK):
+            return path
+    return None
+
 class TestRunner(unittest.TestCase):
 
    def setUp(self):
@@ -29,7 +38,6 @@ class TestRunner(unittest.TestCase):
            forks=1,
            background=0,
            pattern='all',
-           verbose=True,
        )
        self.cwd = os.getcwd()
        self.test_dir = os.path.join(self.cwd, 'test')
@@ -66,14 +74,16 @@ class TestRunner(unittest.TestCase):
        # when using nosetests this will only show up on failure
        # which is pretty useful
        print "RESULTS=%s" % results
-       assert "127.0.0.1" in results['contacted']
-       return results['contacted']['127.0.0.1'] 
+       assert "127.0.0.2" in results['contacted']
+       return results['contacted']['127.0.0.2'] 
 
    def test_ping(self):
        result = self._run('ping',[])
        assert "ping" in result
 
    def test_facter(self):
+       if not get_binary("facter"):
+           raise SkipTest
        result = self._run('facter',[])
        assert "hostname" in result
 
@@ -168,12 +178,13 @@ class TestRunner(unittest.TestCase):
        # almost every time so changed is always true, this just tests that
        # rewriting the file is ok
        result = self._run('setup', [ "metadata=%s" % output, "a=2", "b=3", "c=4" ])
+       print "RAW RESULT=%s" % result
        assert 'md5sum' in result
 
    def test_async(self):
        # test async launch and job status
        # of any particular module
-       result = self._run('command', [ "/bin/sleep", "3" ], background=20)
+       result = self._run('command', [ get_binary("sleep"), "3" ], background=20)
        assert 'ansible_job_id' in result
        assert 'started' in result
        jid = result['ansible_job_id']
@@ -189,7 +200,16 @@ class TestRunner(unittest.TestCase):
        assert 'stdout' in result
        assert result['ansible_job_id'] == jid
 
+   def test_fetch(self):
+       input = self._get_test_file('sample.j2')
+       output = os.path.join(self.stage_dir, '127.0.0.2', input)
+       result = self._run('fetch', [ "src=%s" % input, "dest=%s" % self.stage_dir ])
+       assert os.path.exists(output)
+       assert open(input).read() == open(output).read()
+
    def test_yum(self):
+       if not get_binary("yum"):
+           raise SkipTest
        result = self._run('yum', [ "list=repos" ])
        assert 'failed' not in result
 

test/ansible_hosts

@@ -1,2 +1,2 @@
 [somegroup]
-127.0.0.1
+127.0.0.2

test/inventory_api.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import json
+import sys
+
+from optparse import OptionParser
+
+parser = OptionParser()
+parser.add_option('-l', '--list', default=False, dest="list_hosts", action="store_true")
+parser.add_option('-H', '--host', default=None, dest="host")
+parser.add_option('-e', '--extra-vars', default=None, dest="extra")
+
+options, args = parser.parse_args()
+
+systems = {
+    "ungouped": [ "jupiter", "saturn" ],
+    "greek": [ "zeus", "hera", "poseidon" ],
+    "norse": [ "thor", "odin", "loki" ]
+}
+
+variables = {
+    "thor": {
+        "hammer": True
+        }
+}
+
+if options.list_hosts == True:
+    print json.dumps(systems)
+    sys.exit(0)
+
+if options.host is not None:
+    if options.extra:
+        k,v = options.extra.split("=")
+        variables[options.host][k] = v
+    print json.dumps(variables[options.host])
+    sys.exit(0)
+
+parser.print_help()
+sys.exit(1)

test/playbook1.events

@@ -1,313 +0,0 @@
-{
-    "events": [
-        "start", 
-        [
-            "play start", 
-            [
-                "all"
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "answer": "Wuh, I think so, Brain, but if we didn't have ears, we'd look like weasels.", 
-                    "changed": true, 
-                    "metadata": "/etc/ansible/setup", 
-                    "port": "5150", 
-                    "written": "/etc/ansible/setup"
-                }
-            ]
-        ], 
-        [
-            "import", 
-            [
-                "127.0.0.1", 
-                "/home/mdehaan/ansible/test/common_vars.yml"
-            ]
-        ], 
-        [
-            "import", 
-            [
-                "127.0.0.1", 
-                "/home/mdehaan/ansible/test/CentOS.yml"
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "answer": "Wuh, I think so, Brain, but if we didn't have ears, we'd look like weasels.", 
-                    "changed": true, 
-                    "cow": "moo", 
-                    "duck": "quack", 
-                    "metadata": "/etc/ansible/setup", 
-                    "port": "5150", 
-                    "testing": "default", 
-                    "written": "/etc/ansible/setup"
-                }
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "test basic success command", 
-                false
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "cmd": [
-                        "/bin/true"
-                    ], 
-                    "rc": 0, 
-                    "stderr": "", 
-                    "stdout": ""
-                }
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "test basic success command 2", 
-                false
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "cmd": [
-                        "/bin/true"
-                    ], 
-                    "rc": 0, 
-                    "stderr": "", 
-                    "stdout": ""
-                }
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "test basic shell, plus two ways to dereference a variable", 
-                false
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "cmd": "echo $HOME 5150 5150 ", 
-                    "rc": 0, 
-                    "stderr": "", 
-                    "stdout": "/root 5150 5150"
-                }
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "test vars_files imports", 
-                false
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "cmd": "echo quack moo default ", 
-                    "rc": 0, 
-                    "stderr": "", 
-                    "stdout": "quack moo default"
-                }
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "test copy", 
-                false
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "changed": true, 
-                    "group": "root", 
-                    "mode": 420, 
-                    "path": "/tmp/ansible_test_data_copy.out", 
-                    "state": "file", 
-                    "user": "root"
-                }
-            ]
-        ], 
-        [
-            "notify", 
-            [
-                "127.0.0.1", 
-                "on change 1"
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "test template", 
-                false
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "changed": true, 
-                    "group": "root", 
-                    "mode": 420, 
-                    "path": "/tmp/ansible_test_data_template.out", 
-                    "state": "file", 
-                    "user": "root"
-                }
-            ]
-        ], 
-        [
-            "notify", 
-            [
-                "127.0.0.1", 
-                "on change 1"
-            ]
-        ], 
-        [
-            "notify", 
-            [
-                "127.0.0.1", 
-                "on change 2"
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "async poll test", 
-                false
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "started": 1
-                }
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "started": 1
-                }
-            ]
-        ], 
-        [
-            "async poll", 
-            [
-                "127.0.0.1"
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "cmd": "sleep 5 ", 
-                    "finished": 1, 
-                    "rc": 0, 
-                    "stderr": "", 
-                    "stdout": ""
-                }
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "this should be skipped", 
-                false
-            ]
-        ], 
-        [
-            "skipped", 
-            [
-                "127.0.0.1"
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "on change 1", 
-                true
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "cmd": "echo 'this should fire once' ", 
-                    "rc": 0, 
-                    "stderr": "", 
-                    "stdout": "this should fire once"
-                }
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "cmd": "echo 'this should fire once'  ", 
-                    "rc": 0, 
-                    "stderr": "", 
-                    "stdout": "this should fire once"
-                }
-            ]
-        ], 
-        [
-            "task start", 
-            [
-                "on change 2", 
-                true
-            ]
-        ], 
-        [
-            "ok", 
-            [
-                "127.0.0.1", 
-                {
-                    "cmd": "echo 'this should fire once also' ", 
-                    "rc": 0, 
-                    "stderr": "", 
-                    "stdout": "this should fire once also"
-                }
-            ]
-        ]
-    ], 
-    "results": {
-        "127.0.0.1": {
-            "changed": 2, 
-            "failures": 0, 
-            "ok": 12, 
-            "skipped": 1, 
-            "unreachable": 0
-        }
-    }
-}

test/simple_hosts

@@ -0,0 +1,12 @@
+jupiter
+saturn
+
+[greek]
+zeus
+hera:3000
+poseidon
+
+[norse]
+thor
+odin
+loki

test/yaml_hosts

@@ -0,0 +1,30 @@
+---
+
+- jupiter
+- host: saturn
+  vars:
+    moon: titan
+    moon2: enceladus
+
+- zeus
+
+- group: greek
+  hosts:
+  - zeus
+  - hera
+  - poseidon
+  vars:
+  - ansible_ssh_port: 3000
+  - ntp_server: olympus.example.com
+
+- group: norse
+  hosts:
+  - host: thor
+    vars:
+    - hammer: True
+  - odin
+  - loki
+
+- group: multiple
+  hosts:
+  - saturn