Adding unittest for space preservation within quotes when splitting args

Also adds a new unicode integration test

.gitignore

@@ -35,6 +35,7 @@ docsite/searchindex.js
 docsite/htmlout
 # deb building stuff...
 debian/
+deb-build
 # Vim swap files
 *.swp
 *.swo

CHANGELOG.md

@@ -1,7 +1,68 @@
 Ansible Changes By Release
 ==========================
 
-## 1.6 "And the Cradle Will Rock" - Active Development
+## 1.6.10 "And the Cradle Will Rock" - Jul 25, 2014
+
+- Fixes an issue with the copy module when copying a directory that fails when changing file attributes and the target file already exists
+- Improved unicode handling when splitting args
+
+## 1.6.9 "And the Cradle Will Rock" - Jul 24, 2014
+
+- Further improvements to module parameter parsing to address additional regressions caused by security fixes
+
+## 1.6.8 "And the Cradle Will Rock" - Jul 22, 2014
+
+- Corrects a regression in the way shell and command parameters were being parsed
+
+## 1.6.7 "And the Cradle Will Rock" - Jul 21, 2014
+
+- Security fixes:
+  * Strip lookup calls out of inventory variables and clean unsafe data
+    returned from lookup plugins (CVE-2014-4966)
+  * Make sure vars don't insert extra parameters into module args and prevent
+    duplicate params from superseding previous params (CVE-2014-4967)
+
+## 1.6.6 "And the Cradle Will Rock" - Jul 01, 2014
+
+- Security updates to further protect against the incorrect execution of untrusted data
+
+## 1.6.5 "And the Cradle Will Rock" - Jun 25, 2014
+
+- Additional tweaks to prevent the incorrect execution of untrusted data
+
+## 1.6.4 "And the Cradle Will Rock" - Jun 25, 2014
+
+- Security update to prevent local operations from executing as the result of specifically crafted untrusted data
+
+## 1.6.3 "And the Cradle Will Rock" - Jun 09, 2014
+
+- Corrects a regression where handlers were run across all hosts, not just those that triggered the handler.
+- Fixed a bug in which modules did not support properly moving a file atomically when su was in use.
+- Fixed two bugs related to symlinks with directories when using the file module.
+- Fixed a bug related to MySQL master replication syntax.
+- Corrects a regression in the order of variable merging done by the internal runner code.
+- Various other minor bug fixes.
+
+## 1.6.2 "And the Cradle Will Rock" - May 23, 2014
+ 
+- If an improper locale is specified, core modules will now automatically revert to using the 'C' locale.
+- Modules using the fetch_url utility will now obey proxy environment variables.
+- The SSL validation step in fetch_url will likewise obey proxy settings, however only proxies using the http protocol are supported.
+- Fixed multiple bugs in docker module related to version changes upstream.
+- Fixed a bug in the ec2_group module where egress rules were lost when a VPC was specified.
+- Fixed two bugs in the synchronize module:
+  * a trailing slash might be lost when calculating relative paths, resulting in an incorrect destination.
+  * the sync might use the inventory directory incorrectly instead of the playbook or role directory.
+- Files will now only be chown'd on an atomic move if the src/dest uid/gid do not match.
+
+## 1.6.1 "And the Cradle Will Rock" - May 7, 2014
+
+- Fixed a bug in group_by, where systems were being grouped incorrectly.
+- Fixed a bug where file descriptors may leak to a child process when using accelerate.
+- Fixed a bug in apt_repository triggered when python-apt not being installed/available.
+- Fixed a bug in the apache2_module module, where modules were not being disabled correctly.
+
+## 1.6 "And the Cradle Will Rock" - May 5, 2014
 
 Major features/changes:
 

Makefile

@@ -5,7 +5,9 @@
 #
 # useful targets:
 #   make sdist ---------------- produce a tarball
+#   make srpm ----------------- produce a SRPM
 #   make rpm  ----------------- produce RPMs
+#   make deb-src -------------- produce a DEB source
 #   make deb ------------------ produce a DEB
 #   make docs ----------------- rebuild the manpages (results are checked in)
 #   make tests ---------------- run the tests
@@ -14,7 +16,7 @@
 ########################################################
 # variable section
 
-NAME = "ansible"
+NAME = ansible
 OS = $(shell uname -s)
 
 # Manpages are currently built with asciidoc -- would like to move to markdown
@@ -45,6 +47,29 @@ else
 DATE := $(shell date --utc --date="$(GIT_DATE)" +%Y%m%d%H%M)
 endif
 
+# DEB build parameters
+DEBUILD_BIN ?= debuild
+DEBUILD_OPTS = --source-option="-I"
+DPUT_BIN ?= dput
+DPUT_OPTS ?=
+ifeq ($(OFFICIAL),yes)
+    DEB_RELEASE = 1ppa
+    # Sign OFFICIAL builds using 'DEBSIGN_KEYID'
+    # DEBSIGN_KEYID is required when signing
+    ifneq ($(DEBSIGN_KEYID),)
+        DEBUILD_OPTS += -k$(DEBSIGN_KEYID)
+    endif
+else
+    DEB_RELEASE = 0.git$(DATE)
+    # Do not sign unofficial builds
+    DEBUILD_OPTS += -uc -us
+    DPUT_OPTS += -u
+endif
+DEBUILD = $(DEBUILD_BIN) $(DEBUILD_OPTS)
+DEB_PPA ?= ppa
+# Choose the desired Ubuntu release: lucid precise saucy trusty
+DEB_DIST ?= unstable
+
 # RPM build parameters
 RPMSPECDIR= packaging/rpm
 RPMSPEC = $(RPMSPECDIR)/ansible.spec
@@ -55,6 +80,10 @@ ifeq ($(OFFICIAL),)
 endif
 RPMNVR = "$(NAME)-$(VERSION)-$(RPMRELEASE)$(RPMDIST)"
 
+# MOCK build parameters
+MOCK_BIN ?= mock
+MOCK_CFG ?=
+
 NOSETESTS ?= nosetests
 
 ########################################################
@@ -129,6 +158,20 @@ rpmcommon: $(MANPAGES) sdist
 	@cp dist/*.gz rpm-build/
 	@sed -e 's#^Version:.*#Version: $(VERSION)#' -e 's#^Release:.*#Release: $(RPMRELEASE)%{?dist}#' $(RPMSPEC) >rpm-build/$(NAME).spec
 
+mock-srpm: /etc/mock/$(MOCK_CFG).cfg rpmcommon
+	$(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build/  --buildsrpm --spec rpm-build/$(NAME).spec --sources rpm-build/
+	@echo "#############################################"
+	@echo "Ansible SRPM is built:"
+	@echo rpm-build/*.src.rpm
+	@echo "#############################################"
+
+mock-rpm: /etc/mock/$(MOCK_CFG).cfg mock-srpm
+	$(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build/ --rebuild rpm-build/$(NAME)-*.src.rpm
+	@echo "#############################################"
+	@echo "Ansible RPM is built:"
+	@echo rpm-build/*.noarch.rpm
+	@echo "#############################################"
+
 srpm: rpmcommon
 	@rpmbuild --define "_topdir %(pwd)/rpm-build" \
 	--define "_builddir %{_topdir}" \
@@ -160,12 +203,44 @@ rpm: rpmcommon
 	@echo "#############################################"
 
 debian: sdist
+	@for DIST in $(DEB_DIST) ; do \
+	    mkdir -p deb-build/$${DIST} ; \
+	    tar -C deb-build/$${DIST} -xvf dist/$(NAME)-$(VERSION).tar.gz ; \
+	    cp -a packaging/debian deb-build/$${DIST}/$(NAME)-$(VERSION)/ ; \
+	    sed -ie "s#^$(NAME) (\([^)]*\)) \([^;]*\);#ansible (\1-$(DEB_RELEASE)~$${DIST}) $${DIST};#" deb-build/$${DIST}/$(NAME)-$(VERSION)/debian/changelog ; \
+	done
+
 deb: debian
-	cp -r packaging/debian ./
-	chmod 755 debian/rules
-	fakeroot debian/rules clean
-	fakeroot dh_install
-	fakeroot debian/rules binary
+	@for DIST in $(DEB_DIST) ; do \
+	    (cd deb-build/$${DIST}/$(NAME)-$(VERSION)/ && $(DEBUILD) -b) ; \
+	done
+	@echo "#############################################"
+	@echo "Ansible DEB artifacts:"
+	@for DIST in $(DEB_DIST) ; do \
+	    echo deb-build/$${DIST}/$(NAME)_$(VERSION)-$(DEB_RELEASE)~$${DIST}_amd64.changes ; \
+	done
+	@echo "#############################################"
+
+deb-src: debian
+	@for DIST in $(DEB_DIST) ; do \
+	    (cd deb-build/$${DIST}/$(NAME)-$(VERSION)/ && $(DEBUILD) -S) ; \
+	done
+	@echo "#############################################"
+	@echo "Ansible DEB artifacts:"
+	@for DIST in $(DEB_DIST) ; do \
+	    echo deb-build/$${DIST}/$(NAME)_$(VERSION)-$(DEB_RELEASE)~$${DIST}_source.changes ; \
+	done
+	@echo "#############################################"
+
+deb-upload: deb
+	@for DIST in $(DEB_DIST) ; do \
+	    $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$${DIST}/$(NAME)_$(VERSION)-$(DEB_RELEASE)~$${DIST}_amd64.changes ; \
+	done
+
+deb-src-upload: deb-src
+	@for DIST in $(DEB_DIST) ; do \
+	    $(DPUT_BIN) $(DPUT_OPTS) $(DEB_PPA) deb-build/$${DIST}/$(NAME)_$(VERSION)-$(DEB_RELEASE)~$${DIST}_source.changes ; \
+	done
 
 # for arch or gentoo, read instructions in the appropriate 'packaging' subdirectory directory
 

RELEASES.txt

@@ -1,21 +1,22 @@
 Ansible Releases at a Glance
 ============================
 
-Future
-++++++
-
-1.7    "Summer Nights" -------- SUMMER 2014
-
 Active Development
 ++++++++++++++++++
 
-1.6    "The Cradle Will Rock" - CURRENT DEVEL BRANCH 
+1.7    "Summer Nights" -------- CURRENT DEVEL BRANCH
 
-Previous
-++++++++
-
-=======
-1.6    "The Cradle Will Rock" - NEXT 
+1.6.10 "The Cradle Will Rock" - 07-25-2014
+1.6.9  "The Cradle Will Rock" - 07-24-2014
+1.6.8  "The Cradle Will Rock" - 07-22-2014
+1.6.7  "The Cradle Will Rock" - 07-21-2014
+1.6.6  "The Cradle Will Rock" - 07-01-2014
+1.6.5  "The Cradle Will Rock" - 06-25-2014
+1.6.4  "The Cradle Will Rock" - 06-25-2014
+1.6.3  "The Cradle Will Rock" - 06-09-2014
+1.6.2  "The Cradle Will Rock" - 05-23-2014
+1.6.1  "The Cradle Will Rock" - 05-07-2014
+1.6    "The Cradle Will Rock" - 05-05-2014
 1.5.5  "Love Walks In" -------- 04-18-2014
 1.5.4  "Love Walks In" -------- 04-01-2014
 1.5.3  "Love Walks In" -------- 03-13-2014

VERSION

@@ -1 +1 @@
-1.6
+1.6.10

lib/ansible/__init__.py

@@ -14,5 +14,5 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-__version__ = '1.6'
+__version__ = '1.6.10'
 __author__ = 'Michael DeHaan'

lib/ansible/constants.py

@@ -31,7 +31,7 @@ def mk_boolean(value):
     else:
         return False
 
-def get_config(p, section, key, env_var, default, boolean=False, integer=False, floating=False):
+def get_config(p, section, key, env_var, default, boolean=False, integer=False, floating=False, islist=False):
     ''' return a configuration variable with casting '''
     value = _get_config(p, section, key, env_var, default)
     if boolean:
@@ -40,6 +40,8 @@ def get_config(p, section, key, env_var, default, boolean=False, integer=False,
         return int(value)
     if value and floating:
         return float(value)
+    if value and islist:
+        return [x.strip() for x in value.split(',')]
     return value
 
 def _get_config(p, section, key, env_var, default):
@@ -129,12 +131,12 @@ DEFAULT_SUDO_FLAGS        = get_config(p, DEFAULTS, 'sudo_flags', 'ANSIBLE_SUDO_
 DEFAULT_HASH_BEHAVIOUR    = get_config(p, DEFAULTS, 'hash_behaviour', 'ANSIBLE_HASH_BEHAVIOUR', 'replace')
 DEFAULT_JINJA2_EXTENSIONS = get_config(p, DEFAULTS, 'jinja2_extensions', 'ANSIBLE_JINJA2_EXTENSIONS', None)
 DEFAULT_EXECUTABLE        = get_config(p, DEFAULTS, 'executable', 'ANSIBLE_EXECUTABLE', '/bin/sh')
-DEFAULT_SU_EXE = get_config(p, DEFAULTS, 'su_exe', 'ANSIBLE_SU_EXE', 'su')
-DEFAULT_SU = get_config(p, DEFAULTS, 'su', 'ANSIBLE_SU', False, boolean=True)
-DEFAULT_SU_FLAGS = get_config(p, DEFAULTS, 'su_flags', 'ANSIBLE_SU_FLAGS', '')
-DEFAULT_SU_USER = get_config(p, DEFAULTS, 'su_user', 'ANSIBLE_SU_USER', 'root')
-DEFAULT_ASK_SU_PASS = get_config(p, DEFAULTS, 'ask_su_pass', 'ANSIBLE_ASK_SU_PASS', False, boolean=True)
-DEFAULT_GATHERING = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
+DEFAULT_SU_EXE            = get_config(p, DEFAULTS, 'su_exe', 'ANSIBLE_SU_EXE', 'su')
+DEFAULT_SU                = get_config(p, DEFAULTS, 'su', 'ANSIBLE_SU', False, boolean=True)
+DEFAULT_SU_FLAGS          = get_config(p, DEFAULTS, 'su_flags', 'ANSIBLE_SU_FLAGS', '')
+DEFAULT_SU_USER           = get_config(p, DEFAULTS, 'su_user', 'ANSIBLE_SU_USER', 'root')
+DEFAULT_ASK_SU_PASS       = get_config(p, DEFAULTS, 'ask_su_pass', 'ANSIBLE_ASK_SU_PASS', False, boolean=True)
+DEFAULT_GATHERING         = get_config(p, DEFAULTS, 'gathering', 'ANSIBLE_GATHERING', 'implicit').lower()
 
 DEFAULT_ACTION_PLUGIN_PATH     = get_config(p, DEFAULTS, 'action_plugins',     'ANSIBLE_ACTION_PLUGINS', '/usr/share/ansible_plugins/action_plugins')
 DEFAULT_CALLBACK_PLUGIN_PATH   = get_config(p, DEFAULTS, 'callback_plugins',   'ANSIBLE_CALLBACK_PLUGINS', '/usr/share/ansible_plugins/callback_plugins')
@@ -152,6 +154,7 @@ DEFAULT_UNDEFINED_VAR_BEHAVIOR = get_config(p, DEFAULTS, 'error_on_undefined_var
 HOST_KEY_CHECKING              = get_config(p, DEFAULTS, 'host_key_checking',  'ANSIBLE_HOST_KEY_CHECKING',    True, boolean=True)
 SYSTEM_WARNINGS                = get_config(p, DEFAULTS, 'system_warnings', 'ANSIBLE_SYSTEM_WARNINGS', True, boolean=True)
 DEPRECATION_WARNINGS           = get_config(p, DEFAULTS, 'deprecation_warnings', 'ANSIBLE_DEPRECATION_WARNINGS', True, boolean=True)
+DEFAULT_CALLABLE_WHITELIST     = get_config(p, DEFAULTS, 'callable_whitelist', 'ANSIBLE_CALLABLE_WHITELIST', [], islist=True)
 
 # CONNECTION RELATED
 ANSIBLE_SSH_ARGS               = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', None)

lib/ansible/inventory/dir.py

@@ -36,17 +36,11 @@ class InventoryDirectory(object):
         self.parsers = []
         self.hosts = {}
         self.groups = {}
- 
+
         for i in self.names:
-            
-            if i.endswith("~") or i.endswith(".orig") or i.endswith(".bak"):
-                continue
-            if i.endswith(".ini"):
-                # configuration file for an inventory script
-                continue
-            if i.endswith(".retry"):
-                # this file is generated on a failed playbook and should only be
-                # used when run specifically
+
+            # Skip files that end with certain extensions or characters
+            if any(i.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")):
                 continue
             # Skip hidden files
             if i.startswith('.') and not i.startswith('./'):

lib/ansible/inventory/script.py

@@ -49,7 +49,7 @@ class InventoryScript(object):
     def _parse(self, err):
 
         all_hosts = {}
-        self.raw  = utils.parse_json(self.data)
+        self.raw  = utils.parse_json(self.data, from_remote=True)
         all       = Group('all')
         groups    = dict(all=all)
         group     = None

lib/ansible/module_utils/basic.py

@@ -43,6 +43,7 @@ BOOLEANS = BOOLEANS_TRUE + BOOLEANS_FALSE
 # of an ansible module. The source of this common code lives
 # in lib/ansible/module_common.py
 
+import locale
 import os
 import re
 import pipes
@@ -165,7 +166,6 @@ def load_platform_subclass(cls, *args, **kwargs):
 
     return super(cls, subclass).__new__(subclass)
 
-
 class AnsibleModule(object):
 
     def __init__(self, argument_spec, bypass_checks=False, no_log=False,
@@ -182,6 +182,7 @@ class AnsibleModule(object):
         self.supports_check_mode = supports_check_mode
         self.check_mode = False
         self.no_log = no_log
+        self.cleanup_files = []
         
         self.aliases = {}
         
@@ -190,6 +191,10 @@ class AnsibleModule(object):
                 if k not in self.argument_spec:
                     self.argument_spec[k] = v
 
+        # check the locale as set by the current environment, and
+        # reset to LANG=C if it's an invalid/unavailable locale
+        self._check_locale()
+
         (self.params, self.args) = self._load_params()
 
         self._legal_inputs = ['CHECKMODE', 'NO_LOG']
@@ -560,6 +565,24 @@ class AnsibleModule(object):
             kwargs['state'] = 'absent'
         return kwargs
 
+    def _check_locale(self):
+        '''
+        Uses the locale module to test the currently set locale
+        (per the LANG and LC_CTYPE environment settings)
+        '''
+        try:
+            # setting the locale to '' uses the default locale
+            # as it would be returned by locale.getdefaultlocale()
+            locale.setlocale(locale.LC_ALL, '')
+        except locale.Error, e:
+            # fallback to the 'C' locale, which may cause unicode
+            # issues but is preferable to simply failing because
+            # of an unknown locale
+            locale.setlocale(locale.LC_ALL, 'C')
+            os.environ['LANG']     = 'C'
+            os.environ['LC_CTYPE'] = 'C'
+        except Exception, e:
+            self.fail_json(msg="An unknown error was encountered while attempting to validate the locale: %s" % e)
 
     def _handle_aliases(self):
         aliases_results = {} #alias:canon
@@ -748,6 +771,8 @@ class AnsibleModule(object):
                 (k, v) = x.split("=",1)
             except Exception, e:
                 self.fail_json(msg="this module requires key=value arguments (%s)" % (items))
+            if k in params:
+                self.fail_json(msg="duplicate parameter: %s (value=%s)" % (k, v))
             params[k] = v
         params2 = json.loads(MODULE_COMPLEX_ARGS)
         params2.update(params)
@@ -898,11 +923,20 @@ class AnsibleModule(object):
     def from_json(self, data):
         return json.loads(data)
 
+    def add_cleanup_file(self, path):
+        if path not in self.cleanup_files:
+            self.cleanup_files.append(path)
+
+    def do_cleanup_files(self):
+        for path in self.cleanup_files:
+            self.cleanup(path)
+
     def exit_json(self, **kwargs):
         ''' return from the module, without error '''
         self.add_path_info(kwargs)
         if not 'changed' in kwargs:
             kwargs['changed'] = False
+        self.do_cleanup_files()
         print self.jsonify(kwargs)
         sys.exit(0)
 
@@ -911,6 +945,7 @@ class AnsibleModule(object):
         self.add_path_info(kwargs)
         assert 'msg' in kwargs, "implementation error -- msg to explain the error is required"
         kwargs['failed'] = True
+        self.do_cleanup_files()
         print self.jsonify(kwargs)
         sys.exit(1)
 
@@ -958,7 +993,7 @@ class AnsibleModule(object):
             self.fail_json(msg='Could not make backup of %s to %s: %s' % (fn, backupdest, e))
         return backupdest
 
-    def cleanup(self,tmpfile):
+    def cleanup(self, tmpfile):
         if os.path.exists(tmpfile):
             try:
                 os.unlink(tmpfile)
@@ -987,6 +1022,18 @@ class AnsibleModule(object):
 
         creating = not os.path.exists(dest)
 
+        try:
+            login_name = os.getlogin()
+        except OSError:
+            # not having a tty can cause the above to fail, so
+            # just get the LOGNAME environment variable instead
+            login_name = os.environ.get('LOGNAME', None)
+
+        # if the original login_name doesn't match the currently
+        # logged-in user, or if the SUDO_USER environment variable
+        # is set, then this user has switched their credentials
+        switched_user = login_name and login_name != pwd.getpwuid(os.getuid())[0] or os.environ.get('SUDO_USER')
+
         try:
             # Optimistically try a rename, solves some corner cases and can avoid useless work, throws exception if not atomic.
             os.rename(src, dest)
@@ -1001,7 +1048,7 @@ class AnsibleModule(object):
                 prefix=".ansible_tmp", dir=dest_dir, suffix=dest_file)
 
             try: # leaves tmp file behind when sudo and  not root
-                if os.getenv("SUDO_USER") and os.getuid() != 0:
+                if switched_user and os.getuid() != 0:
                     # cleanup will happen by 'rm' of tempdir
                     # copy2 will preserve some metadata
                     shutil.copy2(src, tmp_dest.name)
@@ -1010,7 +1057,8 @@ class AnsibleModule(object):
                 if self.selinux_enabled():
                     self.set_context_if_different(
                         tmp_dest.name, context, False)
-                if dest_stat:
+                tmp_stat = os.stat(tmp_dest.name)
+                if dest_stat and (tmp_stat.st_uid != dest_stat.st_uid or tmp_stat.st_gid != dest_stat.st_gid):
                     os.chown(tmp_dest.name, dest_stat.st_uid, dest_stat.st_gid)
                 os.rename(tmp_dest.name, dest)
             except (shutil.Error, OSError, IOError), e:
@@ -1023,7 +1071,7 @@ class AnsibleModule(object):
             umask = os.umask(0)
             os.umask(umask)
             os.chmod(dest, 0666 ^ umask)
-            if os.getenv("SUDO_USER"):
+            if switched_user:
                 os.chown(dest, os.getuid(), os.getgid())
 
         if self.selinux_enabled():

lib/ansible/module_utils/urls.py

@@ -50,6 +50,8 @@ try:
 except:
     HAS_SSL=False
 
+import os
+import re
 import socket
 import tempfile
 
@@ -77,6 +79,58 @@ zKPZsZ2miVGclicJHzm5q080b1p/sZtuKIEZk6vZqEg=
 -----END CERTIFICATE-----
 """
 
+def generic_urlparse(parts):
+    '''
+    Returns a dictionary of url parts as parsed by urlparse,
+    but accounts for the fact that older versions of that
+    library do not support named attributes (ie. .netloc)
+    '''
+    generic_parts = dict()
+    if hasattr(parts, 'netloc'):
+        # urlparse is newer, just read the fields straight
+        # from the parts object
+        generic_parts['scheme']   = parts.scheme
+        generic_parts['netloc']   = parts.netloc
+        generic_parts['path']     = parts.path
+        generic_parts['params']   = parts.params
+        generic_parts['query']    = parts.query
+        generic_parts['fragment'] = parts.fragment
+        generic_parts['username'] = parts.username
+        generic_parts['password'] = parts.password
+        generic_parts['hostname'] = parts.hostname
+        generic_parts['port']     = parts.port
+    else:
+        # we have to use indexes, and then parse out
+        # the other parts not supported by indexing
+        generic_parts['scheme']   = parts[0]
+        generic_parts['netloc']   = parts[1]
+        generic_parts['path']     = parts[2]
+        generic_parts['params']   = parts[3]
+        generic_parts['query']    = parts[4]
+        generic_parts['fragment'] = parts[5]
+        # get the username, password, etc.
+        try:
+            netloc_re = re.compile(r'^((?:\w)+(?::(?:\w)+)?@)?([A-Za-z0-9.-]+)(:\d+)?$')
+            (auth, hostname, port) = netloc_re.match(parts[1])
+            if port:
+                # the capture group for the port will include the ':',
+                # so remove it and convert the port to an integer
+                port = int(port[1:])
+            if auth:
+                # the capture group above inclues the @, so remove it
+                # and then split it up based on the first ':' found
+                auth = auth[:-1]
+                username, password = auth.split(':', 1)
+            generic_parts['username'] = username
+            generic_parts['password'] = password
+            generic_parts['hostname'] = hostnme
+            generic_parts['port']     = port
+        except:
+            generic_parts['username'] = None
+            generic_parts['password'] = None
+            generic_parts['hostname'] = None
+            generic_parts['port']     = None
+    return generic_parts
 
 class RequestWithMethod(urllib2.Request):
     '''
@@ -103,6 +157,7 @@ class SSLValidationHandler(urllib2.BaseHandler):
     http://stackoverflow.com/questions/1087227/validate-ssl-certificates-with-python
     http://techknack.net/python-urllib2-handlers/
     '''
+    CONNECT_COMMAND = "CONNECT %s:%s HTTP/1.0\r\nConnection: close\r\n"
 
     def __init__(self, module, hostname, port):
         self.module = module
@@ -161,13 +216,42 @@ class SSLValidationHandler(urllib2.BaseHandler):
 
         return (tmp_path, paths_checked)
 
+    def validate_proxy_response(self, response, valid_codes=[200]):
+        '''
+        make sure we get back a valid code from the proxy
+        '''
+        try:
+            (http_version, resp_code, msg) = re.match(r'(HTTP/\d\.\d) (\d\d\d) (.*)', response).groups()
+            if int(resp_code) not in valid_codes:
+                raise Exception
+        except:
+            self.module.fail_json(msg='Connection to proxy failed')
+
     def http_request(self, req):
         tmp_ca_cert_path, paths_checked = self.get_ca_certs()
+        https_proxy = os.environ.get('https_proxy')
         try:
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
-            ssl_s.connect((self.hostname, self.port))
-            ssl_s.close()
+            if https_proxy:
+                proxy_parts = generic_urlparse(urlparse.urlparse(https_proxy))
+                s.connect((proxy_parts.get('hostname'), proxy_parts.get('port')))
+                if proxy_parts.get('scheme') == 'http':
+                    s.sendall(self.CONNECT_COMMAND % (self.hostname, self.port))
+                    if proxy_parts.get('username'):
+                        credentials = "%s:%s" % (proxy_parts.get('username',''), proxy_parts.get('password',''))
+                        s.sendall('Proxy-Authorization: Basic %s\r\n' % credentials.encode('base64').strip())
+                    s.sendall('\r\n')
+                    connect_result = s.recv(4096)
+                    self.validate_proxy_response(connect_result)
+                    ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
+                else:
+                    self.module.fail_json(msg='Unsupported proxy scheme: %s. Currently ansible only supports HTTP proxies.' % proxy_parts.get('scheme'))
+            else:
+                s.connect((self.hostname, self.port))
+                ssl_s = ssl.wrap_socket(s, ca_certs=tmp_ca_cert_path, cert_reqs=ssl.CERT_REQUIRED)
+            # close the ssl connection
+            #ssl_s.unwrap()
+            s.close()
         except (ssl.SSLError, socket.error), e:
             # fail if we tried all of the certs but none worked
             if 'connection refused' in str(e).lower():
@@ -207,7 +291,7 @@ def url_argument_spec():
 
 
 def fetch_url(module, url, data=None, headers=None, method=None, 
-              use_proxy=False, force=False, last_mod_time=None, timeout=10):
+              use_proxy=True, force=False, last_mod_time=None, timeout=10):
     '''
     Fetches a file from an HTTP/FTP server using urllib2
     '''
@@ -227,6 +311,8 @@ def fetch_url(module, url, data=None, headers=None, method=None,
     # Get validate_certs from the module params
     validate_certs = module.params.get('validate_certs', True)
 
+    # FIXME: change the following to use the generic_urlparse function
+    #        to remove the indexed references for 'parsed'
     parsed = urlparse.urlparse(url)
     if parsed[0] == 'https':
         if not HAS_SSL and validate_certs:

lib/ansible/playbook/__init__.py

@@ -179,7 +179,46 @@ class PlayBook(object):
 
     # *****************************************************
 
-    def _load_playbook_from_file(self, path, vars={}):
+    def _get_playbook_vars(self, play_ds, existing_vars):
+        '''
+        Gets the vars specified with the play and blends them 
+        with any existing vars that have already been read in
+        '''
+        new_vars = existing_vars.copy()
+        if 'vars' in play_ds:
+            if isinstance(play_ds['vars'], dict):
+                new_vars.update(play_ds['vars'])
+            elif isinstance(play_ds['vars'], list):
+                for v in play_ds['vars']:
+                    new_vars.update(v)
+        return new_vars
+
+    # *****************************************************
+
+    def _get_include_info(self, play_ds, basedir, existing_vars={}):
+        '''
+        Gets any key=value pairs specified with the included file
+        name and returns the merged vars along with the path
+        '''
+        new_vars = existing_vars.copy()
+        tokens = shlex.split(play_ds.get('include', ''))
+        for t in tokens[1:]:
+            (k,v) = t.split("=", 1)
+            new_vars[k] = template(basedir, v, new_vars)
+
+        return (new_vars, tokens[0])
+
+    # *****************************************************
+
+    def _get_playbook_vars_files(self, play_ds, existing_vars_files):
+        new_vars_files = list(existing_vars_files)
+        if 'vars_files' in play_ds:
+            new_vars_files = utils.list_union(new_vars_files, play_ds['vars_files'])
+        return new_vars_files
+
+    # *****************************************************
+
+    def _load_playbook_from_file(self, path, vars={}, vars_files=[]):
         '''
         run top level error checking on playbooks and allow them to include other playbooks.
         '''
@@ -201,36 +240,29 @@ class PlayBook(object):
                 # a playbook (list of plays) decided to include some other list of plays
                 # from another file.  The result is a flat list of plays in the end.
 
-                tokens = shlex.split(play['include'])
+                play_vars = self._get_playbook_vars(play, vars)
+                play_vars_files = self._get_playbook_vars_files(play, vars_files)
+                inc_vars, inc_path = self._get_include_info(play, basedir, play_vars)
+                play_vars.update(inc_vars)
 
-                incvars = vars.copy()
-                if 'vars' in play:
-                    if isinstance(play['vars'], dict):
-                        incvars.update(play['vars'])
-                    elif isinstance(play['vars'], list):
-                        for v in play['vars']:
-                            incvars.update(v)
-
-                # allow key=value parameters to be specified on the include line
-                # to set variables
-
-                for t in tokens[1:]:
-
-                    (k,v) = t.split("=", 1)
-                    incvars[k] = template(basedir, v, incvars)
-
-                included_path = utils.path_dwim(basedir, template(basedir, tokens[0], incvars))
-                (plays, basedirs) = self._load_playbook_from_file(included_path, incvars)
+                included_path = utils.path_dwim(basedir, template(basedir, inc_path, play_vars))
+                (plays, basedirs) = self._load_playbook_from_file(included_path, vars=play_vars, vars_files=play_vars_files)
                 for p in plays:
                     # support for parameterized play includes works by passing
                     # those variables along to the subservient play
                     if 'vars' not in p:
                         p['vars'] = {}
                     if isinstance(p['vars'], dict):
-                        p['vars'].update(incvars)
+                        p['vars'].update(play_vars)
                     elif isinstance(p['vars'], list):
                         # nobody should really do this, but handle vars: a=1 b=2
-                        p['vars'].extend([{k:v} for k,v in incvars.iteritems()])
+                        p['vars'].extend([{k:v} for k,v in play_vars.iteritems()])
+                    elif p['vars'] == None:
+                        # someone specified an empty 'vars:', so reset
+                        # it to the vars we currently have
+                        p['vars'] = play_vars.copy()
+                    # now add in the vars_files
+                    p['vars_files'] = utils.list_union(p.get('vars_files', []), play_vars_files)
 
                 accumulated_plays.extend(plays)
                 play_basedirs.extend(basedirs)
@@ -320,14 +352,14 @@ class PlayBook(object):
     def _trim_unavailable_hosts(self, hostlist=[]):
         ''' returns a list of hosts that haven't failed and aren't dark '''
 
-        return [ h for h in self.inventory.list_hosts(hostlist) if (h not in self.stats.failures) and (h not in self.stats.dark)]
+        return [ h for h in hostlist if (h not in self.stats.failures) and (h not in self.stats.dark)]
 
     # *****************************************************
 
     def _run_task_internal(self, task):
         ''' run a particular module step in a playbook '''
 
-        hosts = self._trim_unavailable_hosts(task.play._play_hosts)
+        hosts = self._trim_unavailable_hosts(self.inventory.list_hosts(task.play._play_hosts))
         self.inventory.restrict_to(hosts)
 
         runner = ansible.runner.Runner(
@@ -505,6 +537,7 @@ class PlayBook(object):
 
         # push any variables down to the system
         setup_results = ansible.runner.Runner(
+            basedir=self.basedir,
             pattern=play.hosts,
             module_name='setup',
             module_args={},

lib/ansible/playbook/play.py

@@ -294,26 +294,24 @@ class Play(object):
                             else:
                                 self.included_roles.append(dep)
 
+                        def _merge_conditional(cur_conditionals, new_conditionals):
+                            if isinstance(new_conditionals, (basestring, bool)):
+                                cur_conditionals.append(new_conditionals)
+                            elif isinstance(new_conditionals, list):
+                                cur_conditionals.extend(new_conditionals)
+
                         # pass along conditionals from roles to dep roles
-                        if type(role) is dict:
-                            if 'when' in passed_vars:
-                                if 'when' in dep_vars:
-                                    tmpcond = []
+                        passed_when = passed_vars.get('when')
+                        role_when = role_vars.get('when')
+                        dep_when = dep_vars.get('when')
 
-                                    if type(passed_vars['when']) is str:
-                                        tmpcond.append(passed_vars['when'])
-                                    elif type(passed_vars['when']) is list:
-                                        tmpcond += passed_vars['when']
+                        tmpcond = []
+                        _merge_conditional(tmpcond, passed_when)
+                        _merge_conditional(tmpcond, role_when)
+                        _merge_conditional(tmpcond, dep_when)
 
-                                    if type(dep_vars['when']) is str:
-                                        tmpcond.append(dep_vars['when'])
-                                    elif type(dep_vars['when']) is list:
-                                        tmpcond += dep_vars['when']
-
-                                    if len(tmpcond) > 0:
-                                        dep_vars['when'] = tmpcond
-                                else:
-                                    dep_vars['when'] = passed_vars['when']
+                        if len(tmpcond) > 0:
+                            dep_vars['when'] = tmpcond
 
                         self._build_role_dependencies([dep], dep_stack, passed_vars=dep_vars, level=level+1)
                         dep_stack.append([dep,dep_path,dep_vars,dep_defaults_data])
@@ -528,7 +526,6 @@ class Play(object):
 
             if 'include' in x:
                 tokens = shlex.split(str(x['include']))
-                items = ['']
                 included_additional_conditions = list(additional_conditions)
                 include_vars = {}
                 for k in x:
@@ -565,30 +562,31 @@ class Play(object):
                     task_vars = utils.combine_vars(task_vars, x['vars'])
 
                 if 'when' in x:
-                    included_additional_conditions.append(x['when'])
+                    if isinstance(x['when'], (basestring, bool)):
+                        included_additional_conditions.append(x['when'])
+                    elif isinstance(x['when'], list):
+                        included_additional_conditions.extend(x['when'])
 
                 new_role = None
                 if 'role_name' in x:
                     new_role = x['role_name']
 
-                for item in items:
-                    mv = task_vars.copy()
-                    mv['item'] = item
-                    for t in tokens[1:]:
-                        (k,v) = t.split("=", 1)
-                        mv[k] = template(self.basedir, v, mv)
-                    dirname = self.basedir
-                    if original_file:
-                        dirname = os.path.dirname(original_file)
-                    include_file = template(dirname, tokens[0], mv)
-                    include_filename = utils.path_dwim(dirname, include_file)
-                    data = utils.parse_yaml_from_file(include_filename, vault_password=self.vault_password)
-                    if 'role_name' in x and data is not None:
-                        for y in data:
-                            if isinstance(y, dict) and 'include' in y:
-                                y['role_name'] = new_role
-                    loaded = self._load_tasks(data, mv, default_vars, included_sudo_vars, list(included_additional_conditions), original_file=include_filename, role_name=new_role)
-                    results += loaded
+                mv = task_vars.copy()
+                for t in tokens[1:]:
+                    (k,v) = t.split("=", 1)
+                    mv[k] = template(self.basedir, v, mv)
+                dirname = self.basedir
+                if original_file:
+                    dirname = os.path.dirname(original_file)
+                include_file = template(dirname, tokens[0], mv)
+                include_filename = utils.path_dwim(dirname, include_file)
+                data = utils.parse_yaml_from_file(include_filename, vault_password=self.vault_password)
+                if 'role_name' in x and data is not None:
+                    for y in data:
+                        if isinstance(y, dict) and 'include' in y:
+                            y['role_name'] = new_role
+                loaded = self._load_tasks(data, mv, default_vars, included_sudo_vars, list(included_additional_conditions), original_file=include_filename, role_name=new_role)
+                results += loaded
             elif type(x) == dict:
                 task = Task(
                     self, x,
@@ -785,7 +783,7 @@ class Play(object):
             """ update a host's varscache with new var data """
 
             data = utils.combine_vars(inject, data)
-            self.playbook.VARS_CACHE[host].update(data)
+            self.playbook.VARS_CACHE[host] = utils.combine_vars(self.playbook.VARS_CACHE.get(host, {}), data)
             self.playbook.callbacks.on_import_for_host(host, filename4)
 
         def process_files(filename, filename2, filename3, filename4, host=None):
@@ -855,5 +853,4 @@ class Play(object):
 
         # finally, update the VARS_CACHE for the host, if it is set
         if host is not None:
-            self.playbook.VARS_CACHE[host].update(self.vars)
             self.playbook.VARS_CACHE[host].update(self.playbook.extra_vars)

lib/ansible/playbook/task.py

@@ -256,6 +256,10 @@ class Task(object):
         # make first_available_file accessable to Runner code
         if self.first_available_file:
             self.module_vars['first_available_file'] = self.first_available_file
+            # make sure that the 'item' variable is set when using
+            # first_available_file (issue #8220)
+            if 'item' not in self.module_vars:
+                self.module_vars['item'] = ''
 
         if self.items_lookup_plugin is not None:
             self.module_vars['items_lookup_plugin'] = self.items_lookup_plugin

lib/ansible/runner/__init__.py

@@ -31,6 +31,7 @@ import sys
 import pipes
 import jinja2
 import subprocess
+import shlex
 import getpass
 
 import ansible.constants as C
@@ -46,6 +47,7 @@ import connection
 from return_data import ReturnData
 from ansible.callbacks import DefaultRunnerCallbacks, vv
 from ansible.module_common import ModuleReplacer
+from ansible.utils.splitter import split_args
 
 module_replacer = ModuleReplacer(strip_comments=False)
 
@@ -92,7 +94,7 @@ class HostVars(dict):
 
     def __getitem__(self, host):
         if host not in self.lookup:
-            result = self.inventory.get_variables(host, vault_password=self.vault_password)
+            result = self.inventory.get_variables(host, vault_password=self.vault_password).copy()
             result.update(self.vars_cache.get(host, {}))
             self.lookup[host] = result
         return self.lookup[host]
@@ -394,6 +396,34 @@ class Runner(object):
 
         return actual_user
 
+    def _count_module_args(self, args, allow_dupes=False):
+        '''
+        Count the number of k=v pairs in the supplied module args. This is
+        basically a specialized version of parse_kv() from utils with a few
+        minor changes.
+        '''
+        options = {}
+        if args is not None:
+            try:
+                vargs = split_args(args)
+            except Exception, e:
+                if "unbalanced jinja2 block or quotes" in str(e):
+                    raise errors.AnsibleError("error parsing argument string '%s', try quoting the entire line." % args)
+                else:
+                    raise
+            for x in vargs:
+                quoted = x.startswith('"') and x.endswith('"') or x.startswith("'") and x.endswith("'")
+                if "=" in x and not quoted:
+                    k, v = x.split("=",1)
+                    is_shell_module = self.module_name in ('command', 'shell')
+                    is_shell_param = k in ('creates', 'removes', 'chdir', 'executable')
+                    if k in options and not allow_dupes:
+                        if not(is_shell_module and not is_shell_param):
+                            raise errors.AnsibleError("a duplicate parameter was found in the argument string (%s)" % k)
+                    if is_shell_module and is_shell_param or not is_shell_module:
+                        options[k] = v
+        return len(options)
+
 
     # *****************************************************
 
@@ -507,7 +537,7 @@ class Runner(object):
                 cmd2 = "rm -rf %s >/dev/null 2>&1" % tmp
                 self._low_level_exec_command(conn, cmd2, tmp, sudoable=False)
 
-        data = utils.parse_json(res['stdout'])
+        data = utils.parse_json(res['stdout'], from_remote=True)
         if 'parsed' in data and data['parsed'] == False:
             data['msg'] += res['stderr']
         return ReturnData(conn=conn, result=data)
@@ -566,11 +596,14 @@ class Runner(object):
 
         # merge the VARS and SETUP caches for this host
         combined_cache = self.setup_cache.copy()
-        combined_cache.get(host, {}).update(self.vars_cache.get(host, {}))
+        combined_cache.setdefault(host, {}).update(self.vars_cache.get(host, {}))
         hostvars = HostVars(combined_cache, self.inventory, vault_password=self.vault_pass)
 
         # use combined_cache and host_variables to template the module_vars
-        module_vars_inject = utils.combine_vars(combined_cache.get(host, {}), host_variables)
+        # we update the inject variables with the data we're about to template
+        # since some of the variables we'll be replacing may be contained there too
+        module_vars_inject = utils.combine_vars(host_variables, combined_cache.get(host, {}))
+        module_vars_inject = utils.combine_vars(self.module_vars, module_vars_inject)
         module_vars = template.template(self.basedir, self.module_vars, module_vars_inject)
 
         inject = {}
@@ -609,6 +642,9 @@ class Runner(object):
             items_terms = self.module_vars.get('items_lookup_terms', '')
             items_terms = template.template(basedir, items_terms, inject)
             items = utils.plugins.lookup_loader.get(items_plugin, runner=self, basedir=basedir).run(items_terms, inject=inject)
+            # strip out any jinja2 template syntax within
+            # the data returned by the lookup plugin
+            items = utils._clean_data_struct(items, from_remote=True)
             if type(items) != list:
                 raise errors.AnsibleError("lookup plugins have to return a list: %r" % items)
 
@@ -820,7 +856,20 @@ class Runner(object):
 
         # render module_args and complex_args templates
         try:
+            # When templating module_args, we need to be careful to ensure
+            # that no variables inadvertantly (or maliciously) add params
+            # to the list of args. We do this by counting the number of k=v
+            # pairs before and after templating.
+            num_args_pre = self._count_module_args(module_args, allow_dupes=True)
             module_args = template.template(self.basedir, module_args, inject, fail_on_undefined=self.error_on_undefined_vars)
+            num_args_post = self._count_module_args(module_args)
+            if num_args_pre != num_args_post:
+                raise errors.AnsibleError("A variable inserted a new parameter into the module args. " + \
+                                          "Be sure to quote variables if they contain equal signs (for example: \"{{var}}\").")
+            # And we also make sure nothing added in special flags for things
+            # like the command/shell module (ie. #USE_SHELL)
+            if '#USE_SHELL' in module_args:
+                raise errors.AnsibleError("A variable tried to add #USE_SHELL to the module arguments.")
             complex_args = template.template(self.basedir, complex_args, inject, fail_on_undefined=self.error_on_undefined_vars)
         except jinja2.exceptions.UndefinedError, e:
             raise errors.AnsibleUndefinedVariable("One or more undefined variables: %s" % str(e))
@@ -890,7 +939,7 @@ class Runner(object):
                 if (module_name == 'async_status' and "finished" in data) or module_name != 'async_status':
                     if changed_when is not None and 'skipped' not in data:
                         data['changed'] = utils.check_conditional(changed_when, self.basedir, inject, fail_on_undefined=self.error_on_undefined_vars)
-                    if failed_when is not None:
+                    if failed_when is not None and 'skipped' not in data:
                         data['failed_when_result'] = data['failed'] = utils.check_conditional(failed_when, self.basedir, inject, fail_on_undefined=self.error_on_undefined_vars)
 
             if is_chained:

lib/ansible/runner/action_plugins/add_host.py

@@ -77,7 +77,7 @@ class ActionModule(object):
         
         groupnames = args.get('groupname', args.get('groups', args.get('group', ''))) 
         # add it to the group if that was specified
-        if groupnames != '':
+        if groupnames:
             for group_name in groupnames.split(","):
                 group_name = group_name.strip()
                 if not inventory.get_group(group_name):

lib/ansible/runner/action_plugins/assemble.py

@@ -122,14 +122,25 @@ class ActionModule(object):
                 self.runner._low_level_exec_command(conn, "chmod a+r %s" % xfered, tmp)
 
             # run the copy module
-            module_args = "%s src=%s dest=%s original_basename=%s" % (module_args, pipes.quote(xfered), pipes.quote(dest), pipes.quote(os.path.basename(src)))
+            new_module_args = dict(
+                src=xfered,
+                dest=dest,
+                original_basename=os.path.basename(src),
+            )
+            module_args_tmp = utils.merge_module_args(module_args, new_module_args)
 
             if self.runner.noop_on_check(inject):
                 return ReturnData(conn=conn, comm_ok=True, result=dict(changed=True), diff=dict(before_header=dest, after_header=src, after=resultant))
             else:
-                res = self.runner._execute_module(conn, tmp, 'copy', module_args, inject=inject)
+                res = self.runner._execute_module(conn, tmp, 'copy', module_args_tmp, inject=inject)
                 res.diff = dict(after=resultant)
                 return res
         else:
-            module_args = "%s src=%s dest=%s original_basename=%s" % (module_args, pipes.quote(xfered), pipes.quote(dest), pipes.quote(os.path.basename(src)))
-            return self.runner._execute_module(conn, tmp, 'file', module_args, inject=inject)
+            new_module_args = dict(
+                src=xfered,
+                dest=dest,
+                original_basename=os.path.basename(src),
+            )
+            module_args_tmp = utils.merge_module_args(module_args, new_module_args)
+
+            return self.runner._execute_module(conn, tmp, 'file', module_args_tmp, inject=inject)

lib/ansible/runner/action_plugins/copy.py

@@ -238,11 +238,16 @@ class ActionModule(object):
 
                 # src and dest here come after original and override them
                 # we pass dest only to make sure it includes trailing slash in case of recursive copy
-                module_args_tmp = "%s src=%s dest=%s original_basename=%s" % (module_args,
-                                  pipes.quote(tmp_src), pipes.quote(dest), pipes.quote(source_rel))
+                new_module_args = dict(
+                    src=tmp_src,
+                    dest=dest,
+                    original_basename=source_rel
+                )
 
                 if self.runner.no_log:
-                    module_args_tmp = "%s NO_LOG=True" % module_args_tmp
+                    new_module_args['NO_LOG'] = True
+
+                module_args_tmp = utils.merge_module_args(module_args, new_module_args)
 
                 module_return = self.runner._execute_module(conn, tmp_path, 'copy', module_args_tmp, inject=inject, complex_args=complex_args, delete_remote_tmp=delete_remote_tmp)
                 module_executed = True
@@ -260,12 +265,17 @@ class ActionModule(object):
                 tmp_src = tmp_path + source_rel
 
                 # Build temporary module_args.
-                module_args_tmp = "%s src=%s original_basename=%s" % (module_args,
-                                  pipes.quote(tmp_src), pipes.quote(source_rel))
+                new_module_args = dict(
+                    src=tmp_src,
+                    dest=dest,
+                    original_basename=source_rel
+                )
                 if self.runner.noop_on_check(inject):
-                    module_args_tmp = "%s CHECKMODE=True" % module_args_tmp
+                    new_module_args['CHECKMODE'] = True
                 if self.runner.no_log:
-                    module_args_tmp = "%s NO_LOG=True" % module_args_tmp
+                    new_module_args['NO_LOG'] = True
+
+                module_args_tmp = utils.merge_module_args(module_args, new_module_args)
 
                 # Execute the file module.
                 module_return = self.runner._execute_module(conn, tmp_path, 'file', module_args_tmp, inject=inject, complex_args=complex_args, delete_remote_tmp=delete_remote_tmp)

lib/ansible/runner/action_plugins/group_by.py

@@ -38,6 +38,8 @@ class ActionModule(object):
         # the group_by module does not need to pay attention to check mode.
         # it always runs.
 
+        # module_args and complex_args have already been templated for the first host.
+        # Use them here only to check that a key argument is provided.
         args = {}
         if complex_args:
             args.update(complex_args)
@@ -68,7 +70,16 @@ class ActionModule(object):
                     break
             if next_host:
                 continue
-            group_name = template.template(self.runner.basedir, args['key'], data)
+
+            # Template original module_args and complex_args from runner for each host.
+            host_module_args = template.template(self.runner.basedir, self.runner.module_args, data)
+            host_complex_args = template.template(self.runner.basedir, self.runner.complex_args, data)
+            host_args  = {}
+            if host_complex_args:
+                host_args.update(host_complex_args)
+            host_args.update(parse_kv(host_module_args))
+
+            group_name = host_args['key']
             group_name = group_name.replace(' ','-')
             if group_name not in groups:
                 groups[group_name] = []

lib/ansible/runner/action_plugins/synchronize.py

@@ -32,11 +32,12 @@ class ActionModule(object):
         if 'vars' in self.inject:
             if '_original_file' in self.inject['vars']:
                 # roles
+                original_path = path
                 path = utils.path_dwim_relative(self.inject['_original_file'], 'files', path, self.runner.basedir)
-            elif 'inventory_dir' in self.inject['vars']:
-                # non-roles
-                abs_dir = os.path.abspath(self.inject['vars']['inventory_dir'])
-                path = os.path.join(abs_dir, path)
+                if original_path and original_path[-1] == '/' and path[-1] != '/':
+                    # make sure the dwim'd path ends in a trailing "/"
+                    # if the original path did
+                    path += '/'
 
         return path
 

lib/ansible/runner/action_plugins/template.py

@@ -117,12 +117,17 @@ class ActionModule(object):
                 self.runner._low_level_exec_command(conn, "chmod a+r %s" % xfered, tmp)
 
             # run the copy module
-            module_args = "%s src=%s dest=%s original_basename=%s" % (module_args, pipes.quote(xfered), pipes.quote(dest), pipes.quote(os.path.basename(source)))
+            new_module_args = dict(
+               src=xfered,
+               dest=dest,
+               original_basename=os.path.basename(source),
+            )
+            module_args_tmp = utils.merge_module_args(module_args, new_module_args)
 
             if self.runner.noop_on_check(inject):
                 return ReturnData(conn=conn, comm_ok=True, result=dict(changed=True), diff=dict(before_header=dest, after_header=source, before=dest_contents, after=resultant))
             else:
-                res = self.runner._execute_module(conn, tmp, 'copy', module_args, inject=inject, complex_args=complex_args)
+                res = self.runner._execute_module(conn, tmp, 'copy', module_args_tmp, inject=inject, complex_args=complex_args)
                 if res.result.get('changed', False):
                     res.diff = dict(before=dest_contents, after=resultant)
                 return res

lib/ansible/runner/action_plugins/unarchive.py

@@ -78,7 +78,12 @@ class ActionModule(object):
         if copy:
             if self.runner.sudo and self.runner.sudo_user != 'root':
                 self.runner._low_level_exec_command(conn, "chmod a+r %s" % tmp_src, tmp)
-            module_args = "%s src=%s original_basename=%s" % (module_args, pipes.quote(tmp_src), pipes.quote(os.path.basename(source)))
+            # Build temporary module_args.
+            new_module_args = dict(
+                src=tmp_src,
+                original_basename=os.path.basename(source),
+            )
+            module_args = utils.merge_module_args(module_args, new_module_args)
         else:
             module_args = "%s original_basename=%s" % (module_args, pipes.quote(os.path.basename(source)))
         return self.runner._execute_module(conn, tmp, 'unarchive', module_args, inject=inject, complex_args=complex_args)

lib/ansible/runner/lookup_plugins/env.py

@@ -16,6 +16,7 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 from ansible import utils, errors
+from ansible.utils import template
 import os
 
 class LookupModule(object):
@@ -25,7 +26,10 @@ class LookupModule(object):
 
     def run(self, terms, inject=None, **kwargs):
 
-        terms = utils.listify_lookup_plugin_terms(terms, self.basedir, inject) 
+        try:
+            terms = template.template(self.basedir, terms, inject)
+        except Exception, e:
+            pass
 
         if isinstance(terms, basestring):
             terms = [ terms ]

lib/ansible/runner/return_data.py

@@ -43,8 +43,7 @@ class ReturnData(object):
         self.diff = diff
 
         if type(self.result) in [ str, unicode ]:
-            self.result = utils.parse_json(self.result)
-
+            self.result = utils.parse_json(self.result, from_remote=True)
 
         if self.host is None:
             raise Exception("host not set")

lib/ansible/utils/__init__.py

@@ -25,10 +25,10 @@ import optparse
 import operator
 from ansible import errors
 from ansible import __version__
-from ansible.utils import template
 from ansible.utils.display_functions import *
 from ansible.utils.plugins import *
 from ansible.callbacks import display
+from ansible.utils.splitter import split_args, unquote
 import ansible.constants as C
 import ast
 import time
@@ -45,13 +45,16 @@ import getpass
 import sys
 import json
 
-#import vault
 from vault import VaultLib
 
 VERBOSITY=0
 
 MAX_FILE_SIZE_FOR_DIFF=1*1024*1024
 
+# caching the compilation of the regex used
+# to check for lookup calls within data
+LOOKUP_REGEX=re.compile(r'lookup\s*\(')
+
 try:
     import json
 except ImportError:
@@ -69,6 +72,11 @@ try:
 except:
     pass
 
+try:
+    import builtin
+except ImportError:
+    import __builtin__ as builtin
+
 KEYCZAR_AVAILABLE=False
 try:
     try:
@@ -194,6 +202,7 @@ def is_changed(result):
     return (result.get('changed', False) in [ True, 'True', 'true'])
 
 def check_conditional(conditional, basedir, inject, fail_on_undefined=False):
+    from ansible.utils import template
 
     if conditional is None or conditional == '':
         return True
@@ -288,6 +297,9 @@ def path_dwim_relative(original, dirname, source, playbook_base, check=True):
     ''' find one file in a directory one level up in a dir named dirname relative to current '''
     # (used by roles code)
 
+    from ansible.utils import template
+
+
     basedir = os.path.dirname(original)
     if os.path.islink(basedir):
         basedir = unfrackpath(basedir)
@@ -309,7 +321,44 @@ def json_loads(data):
 
     return json.loads(data)
 
-def parse_json(raw_data):
+def _clean_data(orig_data, from_remote=False, from_inventory=False):
+    ''' remove template tags from a string '''
+    data = orig_data
+    if isinstance(orig_data, basestring):
+        sub_list = [('{%','{#'), ('%}','#}')]
+        if from_remote or (from_inventory and '{{' in data and LOOKUP_REGEX.search(data)):
+            # if from a remote, we completely disable any jinja2 blocks
+            sub_list.extend([('{{','{#'), ('}}','#}')])
+        for pattern,replacement in sub_list:
+            data = data.replace(pattern, replacement)
+    return data
+
+def _clean_data_struct(orig_data, from_remote=False, from_inventory=False):
+    '''
+    walk a complex data structure, and use _clean_data() to
+    remove any template tags that may exist
+    '''
+    if not from_remote and not from_inventory:
+        raise errors.AnsibleErrors("when cleaning data, you must specify either from_remote or from_inventory")
+    if isinstance(orig_data, dict):
+        data = orig_data.copy()
+        for key in data:
+            new_key = _clean_data_struct(key, from_remote, from_inventory)
+            new_val = _clean_data_struct(data[key], from_remote, from_inventory)
+            if key != new_key:
+                del data[key]
+            data[new_key] = new_val
+    elif isinstance(orig_data, list):
+        data = orig_data[:]
+        for i in range(0, len(data)):
+            data[i] = _clean_data_struct(data[i], from_remote, from_inventory)
+    elif isinstance(orig_data, basestring):
+        data = _clean_data(orig_data, from_remote, from_inventory)
+    else:
+        data = orig_data
+    return data
+
+def parse_json(raw_data, from_remote=False, from_inventory=False):
     ''' this version for module return data only '''
 
     orig_data = raw_data
@@ -318,7 +367,7 @@ def parse_json(raw_data):
     data = filter_leading_non_json_lines(raw_data)
 
     try:
-        return json.loads(data)
+        results = json.loads(data)
     except:
         # not JSON, but try "Baby JSON" which allows many of our modules to not
         # require JSON and makes writing modules in bash much simpler
@@ -328,7 +377,6 @@ def parse_json(raw_data):
         except:
             print "failed to parse json: "+ data
             raise
-
         for t in tokens:
             if "=" not in t:
                 raise errors.AnsibleError("failed to parse: %s" % orig_data)
@@ -343,29 +391,32 @@ def parse_json(raw_data):
             results[key] = value
         if len(results.keys()) == 0:
             return { "failed" : True, "parsed" : False, "msg" : orig_data }
-        return results
 
-def smush_braces(data):
-    ''' smush Jinaj2 braces so unresolved templates like {{ foo }} don't get parsed weird by key=value code '''
-    while '{{ ' in data:
-        data = data.replace('{{ ', '{{')
-    while ' }}' in data:
-        data = data.replace(' }}', '}}')
-    return data
+    if from_remote:
+        results = _clean_data_struct(results, from_remote, from_inventory)
 
-def smush_ds(data):
-    # things like key={{ foo }} are not handled by shlex.split well, so preprocess any YAML we load
-    # so we do not have to call smush elsewhere
-    if type(data) == list:
-        return [ smush_ds(x) for x in data ]
-    elif type(data) == dict:
-        for (k,v) in data.items():
-            data[k] = smush_ds(v)
-        return data
-    elif isinstance(data, basestring):
-        return smush_braces(data)
-    else:
-        return data
+    return results
+
+def merge_module_args(current_args, new_args):
+    '''
+    merges either a dictionary or string of k=v pairs with another string of k=v pairs,
+    and returns a new k=v string without duplicates.
+    '''
+    if not isinstance(current_args, basestring):
+        raise errors.AnsibleError("expected current_args to be a basestring")
+    # we use parse_kv to split up the current args into a dictionary
+    final_args = parse_kv(current_args)
+    if isinstance(new_args, dict):
+        final_args.update(new_args)
+    elif isinstance(new_args, basestring):
+        new_args_kv = parse_kv(new_args)
+        final_args.update(new_args_kv)
+    # then we re-assemble into a string
+    module_args = ""
+    for (k,v) in final_args.iteritems():
+        if isinstance(v, basestring):
+            module_args = "%s=%s %s" % (k, pipes.quote(v), module_args)
+    return module_args.strip()
 
 def parse_yaml(data, path_hint=None):
     ''' convert a yaml string to a data structure.  Also supports JSON, ssssssh!!!'''
@@ -385,7 +436,7 @@ def parse_yaml(data, path_hint=None):
         # else this is pretty sure to be a YAML document
         loaded = yaml.safe_load(data)
 
-    return smush_ds(loaded)
+    return loaded
 
 def process_common_errors(msg, probline, column):
     replaced = probline.replace(" ","")
@@ -567,7 +618,7 @@ def parse_kv(args):
         # attempting to split a unicode here does bad things
         args = args.encode('utf-8')
         try:
-            vargs = shlex.split(args, posix=True)
+            vargs = split_args(args)
         except ValueError, ve:
             if 'no closing quotation' in str(ve).lower():
                 raise errors.AnsibleError("error parsing argument string, try quoting the entire line.")
@@ -577,7 +628,7 @@ def parse_kv(args):
         for x in vargs:
             if "=" in x:
                 k, v = x.split("=",1)
-                options[k]=v
+                options[k] = unquote(v)
     return options
 
 def merge_hash(a, b):
@@ -1005,13 +1056,33 @@ def is_list_of_strings(items):
             return False
     return True
 
+def list_union(a, b):
+    result = []
+    for x in a:
+        if x not in result:
+            result.append(x)
+    for x in b:
+        if x not in result:
+            result.append(x)
+    return result
+
+def list_intersection(a, b):
+    result = []
+    for x in a:
+        if x in b and x not in result:
+            result.append(x)
+    return result
+
 def safe_eval(expr, locals={}, include_exceptions=False):
     '''
-    this is intended for allowing things like:
+    This is intended for allowing things like:
     with_items: a_list_variable
-    where Jinja2 would return a string
-    but we do not want to allow it to call functions (outside of Jinja2, where
-    the env is constrained)
+
+    Where Jinja2 would return a string but we do not want to allow it to
+    call functions (outside of Jinja2, where the env is constrained). If
+    the input data to this function came from an untrusted (remote) source,
+    it should first be run through _clean_data_struct() to ensure the data
+    is further sanitized prior to evaluation.
 
     Based on:
     http://stackoverflow.com/questions/12523516/using-ast-and-whitelists-to-make-pythons-eval-safe
@@ -1023,22 +1094,22 @@ def safe_eval(expr, locals={}, include_exceptions=False):
     # visitor class defined below.
     SAFE_NODES = set(
         (
-            ast.Expression,
-            ast.Compare,
-            ast.Str,
-            ast.List,
-            ast.Tuple,
-            ast.Dict,
-            ast.Call,
-            ast.Load,
+            ast.Add,
             ast.BinOp,
-            ast.UnaryOp,
+            ast.Call,
+            ast.Compare,
+            ast.Dict,
+            ast.Div,
+            ast.Expression,
+            ast.List,
+            ast.Load,
+            ast.Mult,
             ast.Num,
             ast.Name,
-            ast.Add,
+            ast.Str,
             ast.Sub,
-            ast.Mult,
-            ast.Div,
+            ast.Tuple,
+            ast.UnaryOp,
         )
     )
 
@@ -1050,22 +1121,24 @@ def safe_eval(expr, locals={}, include_exceptions=False):
             )
         )
 
-    # builtin functions that are not safe to call
-    INVALID_CALLS = (
-       'classmethod', 'compile', 'delattr', 'eval', 'execfile', 'file',
-       'filter', 'help', 'input', 'object', 'open', 'raw_input', 'reduce',
-       'reload', 'repr', 'setattr', 'staticmethod', 'super', 'type',
-    )
+    filter_list = []
+    for filter in filter_loader.all():
+        filter_list.extend(filter.filters().keys())
+
+    CALL_WHITELIST = C.DEFAULT_CALLABLE_WHITELIST + filter_list
 
     class CleansingNodeVisitor(ast.NodeVisitor):
-        def generic_visit(self, node):
+        def generic_visit(self, node, inside_call=False):
             if type(node) not in SAFE_NODES:
-                #raise Exception("invalid expression (%s) type=%s" % (expr, type(node)))
                 raise Exception("invalid expression (%s)" % expr)
-            super(CleansingNodeVisitor, self).generic_visit(node)
-        def visit_Call(self, call):
-            if call.func.id in INVALID_CALLS:
-                raise Exception("invalid function: %s" % call.func.id)
+            elif isinstance(node, ast.Call):
+                inside_call = True
+            elif isinstance(node, ast.Name) and inside_call:
+                if hasattr(builtin, node.id) and node.id not in CALL_WHITELIST:
+                    raise Exception("invalid function: %s" % node.id)
+            # iterate over all child nodes
+            for child_node in ast.iter_child_nodes(node):
+                self.generic_visit(child_node, inside_call)
 
     if not isinstance(expr, basestring):
         # already templated to a datastructure, perhaps?
@@ -1073,9 +1146,9 @@ def safe_eval(expr, locals={}, include_exceptions=False):
             return (expr, None)
         return expr
 
+    cnv = CleansingNodeVisitor()
     try:
         parsed_tree = ast.parse(expr, mode='eval')
-        cnv = CleansingNodeVisitor()
         cnv.visit(parsed_tree)
         compiled = compile(parsed_tree, expr, 'eval')
         result = eval(compiled, {}, locals)
@@ -1098,6 +1171,8 @@ def safe_eval(expr, locals={}, include_exceptions=False):
 
 def listify_lookup_plugin_terms(terms, basedir, inject):
 
+    from ansible.utils import template
+
     if isinstance(terms, basestring):
         # someone did:
         #    with_items: alist

lib/ansible/utils/splitter.py

@@ -0,0 +1,160 @@
+# (c) 2014 James Cammarata, <jcammarata@ansible.com>
+#
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+def _get_quote_state(token, quote_char):
+    '''
+    the goal of this block is to determine if the quoted string
+    is unterminated in which case it needs to be put back together
+    '''
+    # the char before the current one, used to see if
+    # the current character is escaped
+    prev_char = None
+    for idx, cur_char in enumerate(token):
+        if idx > 0:
+            prev_char = token[idx-1]
+        if cur_char in '"\'':
+            if quote_char:
+                if cur_char == quote_char and prev_char != '\\':
+                    quote_char = None
+            else:
+                quote_char = cur_char
+    return quote_char
+
+def _count_jinja2_blocks(token, cur_depth, open_token, close_token):
+    '''
+    this function counts the number of opening/closing blocks for a
+    given opening/closing type and adjusts the current depth for that
+    block based on the difference
+    '''
+    num_open  = token.count(open_token)
+    num_close = token.count(close_token)
+    if num_open != num_close:
+        cur_depth += (num_open - num_close)
+        if cur_depth < 0:
+            cur_depth = 0
+    return cur_depth
+
+def split_args(args):
+    '''
+    Splits args on whitespace, but intelligently reassembles
+    those that may have been split over a jinja2 block or quotes.
+
+    When used in a remote module, we won't ever have to be concerned about
+    jinja2 blocks, however this function is/will be used in the
+    core portions as well before the args are templated.
+
+    example input: a=b c="foo bar"
+    example output: ['a=b', 'c="foo bar"']
+
+    Basically this is a variation shlex that has some more intelligence for
+    how Ansible needs to use it.
+    '''
+
+    # the list of params parsed out of the arg string
+    # this is going to be the result value when we are donei
+    params = []
+
+    # here we encode the args, so we have a uniform charset to
+    # work with, and split on white space
+    try:
+        args = args.encode('utf-8')
+        do_decode = True
+    except UnicodeDecodeError:
+        do_decode = False
+    tokens = args.split(' ')
+
+    # iterate over the tokens, and reassemble any that may have been
+    # split on a space inside a jinja2 block.
+    # ex if tokens are "{{", "foo", "}}" these go together
+
+    # These variables are used
+    # to keep track of the state of the parsing, since blocks and quotes
+    # may be nested within each other.
+
+    quote_char = None
+    inside_quotes = False
+    print_depth   = 0 # used to count nested jinja2 {{ }} blocks
+    block_depth   = 0 # used to count nested jinja2 {% %} blocks
+    comment_depth = 0 # used to count nested jinja2 {# #} blocks
+
+    # now we loop over each split token, coalescing tokens if the white space
+    # split occurred within quotes or a jinja2 block of some kind
+    for token in tokens:
+
+        # store the previous quoting state for checking later
+        was_inside_quotes = inside_quotes
+        quote_char = _get_quote_state(token, quote_char)
+        inside_quotes = quote_char is not None
+
+        # multiple conditions may append a token to the list of params,
+        # so we keep track with this flag to make sure it only happens once
+        # append means add to the end of the list, don't append means concatenate
+        # it to the end of the last token
+        appended = False
+
+        # if we're inside quotes now, but weren't before, append the token 
+        # to the end of the list, since we'll tack on more to it later
+        # otherwise, if we're inside any jinja2 block, inside quotes, or we were
+        # inside quotes (but aren't now) concat this token to the last param
+        if inside_quotes and not was_inside_quotes:
+            params.append(token)
+            appended = True
+        elif print_depth or block_depth or comment_depth or inside_quotes or was_inside_quotes:
+            params[-1] = "%s %s" % (params[-1], token)
+            appended = True
+
+        # if the number of paired block tags is not the same, the depth has changed, so we calculate that here
+        # and may append the current token to the params (if we haven't previously done so)
+        prev_print_depth = print_depth
+        print_depth = _count_jinja2_blocks(token, print_depth, "{{", "}}")
+        if print_depth != prev_print_depth and not appended:
+            params.append(token)
+            appended = True
+
+        prev_block_depth = block_depth
+        block_depth = _count_jinja2_blocks(token, block_depth, "{%", "%}")
+        if block_depth != prev_block_depth and not appended:
+            params.append(token)
+            appended = True
+
+        prev_comment_depth = comment_depth
+        comment_depth = _count_jinja2_blocks(token, comment_depth, "{#", "#}")
+        if comment_depth != prev_comment_depth and not appended:
+            params.append(token)
+            appended = True
+
+        # finally, if we're at zero depth for all blocks and not inside quotes, and have not
+        # yet appended anything to the list of params, we do so now
+        if not (print_depth or block_depth or comment_depth) and not inside_quotes and not appended and token != '':
+            params.append(token)
+
+    # If we're done and things are not at zero depth or we're still inside quotes,
+    # raise an error to indicate that the args were unbalanced
+    if print_depth or block_depth or comment_depth or inside_quotes:
+        raise Exception("error while splitting arguments, either an unbalanced jinja2 block or quotes")
+
+    # finally, we decode each param back to the unicode it was in the arg string
+    if do_decode:
+        params = [x.decode('utf-8') for x in params]
+    return params
+
+def unquote(data):
+    ''' removes first and last quotes from a string, if the string starts and ends with the same quotes '''
+    if len(data) > 0 and (data[0] == '"' and data[-1] == '"' or data[0] == "'" and data[-1] == "'"):
+        return data[1:-1]
+    return data
+

lib/ansible/utils/template.py

@@ -80,7 +80,6 @@ class Flags:
 
 FILTER_PLUGINS = None
 _LISTRE = re.compile(r"(\w+)\[(\d+)\]")
-JINJA2_OVERRIDE='#jinja2:'
 
 def lookup(name, *args, **kwargs):
     from ansible import utils
@@ -228,16 +227,6 @@ def template_from_file(basedir, path, vars, vault_password=None):
     except:
         raise errors.AnsibleError("unable to read %s" % realpath)
 
-
-    # Get jinja env overrides from template
-    if data.startswith(JINJA2_OVERRIDE):
-        eol = data.find('\n')
-        line = data[len(JINJA2_OVERRIDE):eol]
-        data = data[eol+1:]
-        for pair in line.split(','):
-            (key,val) = pair.split(':')
-            setattr(environment,key.strip(),ast.literal_eval(val.strip()))
-
     environment.template_class = J2Template
     try:
         t = environment.from_string(data)

library/cloud/docker

@@ -384,7 +384,7 @@ class DockerManager:
 
         self.env = None
         if self.module.params.get('env'):
-            self.env = dict(map(lambda x: x.split("="), self.module.params.get('env')))
+            self.env = dict(map(lambda x: x.split("=", 1), self.module.params.get('env')))
 
         # connect to docker server
         docker_url = urlparse(module.params.get('docker_url'))
@@ -507,14 +507,16 @@ class DockerManager:
             running_image, running_tag = self.get_split_image_tag(i['Image'])
             running_command = i['Command'].strip()
 
-            name_matches = (name and name in i['Names'])
+            name_matches = False
+            if i["Names"]:
+                name_matches = (name and name in i['Names'])
             image_matches = (running_image == image)
             tag_matches = (not tag or running_tag == tag)
             # if a container has an entrypoint, `command` will actually equal
             # '{} {}'.format(entrypoint, command)
             command_matches = (not command or running_command.endswith(command))
 
-            if name_matches or (image_matches and tag_matches and command_matches):
+            if name_matches or (name is None and image_matches and tag_matches and command_matches):
                 details = self.client.inspect_container(i['Id'])
                 details = _docker_id_quirk(details)
                 deployed.append(details)
@@ -524,7 +526,7 @@ class DockerManager:
     def get_running_containers(self):
         running = []
         for i in self.get_deployed_containers():
-            if i['State']['Running'] == True and i['State']['Ghost'] == False:
+            if i['State']['Running'] == True and i['State'].get('Ghost', False) == False:
                 running.append(i)
 
         return running

library/cloud/ec2

@@ -175,7 +175,7 @@ options:
   instance_ids:
     version_added: "1.3"
     description:
-      - list of instance ids, currently only used when state='absent'
+      - list of instance ids, currently used for the states 'absent', 'running', and 'stopped'
     required: false
     default: null
     aliases: []
@@ -1025,7 +1025,7 @@ def terminate_instances(module, ec2, instance_ids):
     return (changed, instance_dict_array, terminated_instance_ids)
 
 
-def startstop_instances(module, ec2, instance_ids):
+def startstop_instances(module, ec2, instance_ids, state):
     """
     Starts or stops a list of existing instances
 
@@ -1033,9 +1033,10 @@ def startstop_instances(module, ec2, instance_ids):
     ec2: authenticated ec2 connection object
     instance_ids: The list of instances to start in the form of
       [ {id: <inst-id>}, ..]
+    state: Intended state ("running" or "stopped")
 
     Returns a dictionary of instance information
-    about the instances started.
+    about the instances started/stopped.
 
     If the instance was not able to change state,
     "changed" will be set to False.
@@ -1050,18 +1051,15 @@ def startstop_instances(module, ec2, instance_ids):
     if not isinstance(instance_ids, list) or len(instance_ids) < 1:
         module.fail_json(msg='instance_ids should be a list of instances, aborting')
 
-    dest_state = module.params.get('state')
-    dest_state_ec2 = 'stopped' if dest_state == 'stopped' else 'running'
-
     # Check that our instances are not in the state we want to take them to
     # and change them to our desired state
     running_instances_array = []
     for res in ec2.get_all_instances(instance_ids):
         for inst in res.instances:
-           if not inst.state == dest_state_ec2:
+           if inst.state != state:
                instance_dict_array.append(get_instance_info(inst))
                try:
-                   if dest_state == 'running':
+                   if state == 'running':
                        inst.start()
                    else:
                        inst.stop()
@@ -1070,21 +1068,14 @@ def startstop_instances(module, ec2, instance_ids):
                changed = True
 
     ## Wait for all the instances to finish starting or stopping
-    instids = [ i.id for i in res.instances ]
-    this_res = []
-    num_running = 0
     wait_timeout = time.time() + wait_timeout
-    while wait_timeout > time.time() and num_running < len(instids):
-        res_list = res.connection.get_all_instances(instids)
-        if len(res_list) > 0:
-            this_res = res_list[0]
-            num_running = len([ i for i in this_res.instances if i.state == dest_state_ec2 ])
-        else:
-            # got a bad response of some sort, possibly due to 
-            # stale/cached data. Wait a second and then try again
-            time.sleep(1)
-            continue
-        if wait and num_running < len(instids):
+    while wait and wait_timeout > time.time():
+        matched_instances = []
+        for res in ec2.get_all_instances(instance_ids):
+            for i in res.instances:
+                if i.state == state:
+                    matched_instances.append(i)
+        if len(matched_instances) < len(instance_ids):
             time.sleep(5)
         else:
             break
@@ -1093,10 +1084,7 @@ def startstop_instances(module, ec2, instance_ids):
         # waiting took too long
         module.fail_json(msg = "wait for instances running timeout on %s" % time.asctime())
 
-    for inst in this_res.instances:
-        running_instances_array.append(inst.id)
-
-    return (changed, instance_dict_array, running_instances_array)
+    return (changed, instance_dict_array, instance_ids)
 
 
 def main():
@@ -1147,21 +1135,23 @@ def main():
 
     tagged_instances = [] 
 
-    if module.params.get('state') == 'absent':
+    state = module.params.get('state')
+
+    if state == 'absent':
         instance_ids = module.params.get('instance_ids')
         if not isinstance(instance_ids, list):
             module.fail_json(msg='termination_list needs to be a list of instances to terminate')
 
         (changed, instance_dict_array, new_instance_ids) = terminate_instances(module, ec2, instance_ids)
 
-    elif module.params.get('state') == 'running' or module.params.get('state') == 'stopped':
+    elif state in ('running', 'stopped'):
         instance_ids = module.params.get('instance_ids')
         if not isinstance(instance_ids, list):
             module.fail_json(msg='running list needs to be a list of instances to run: %s' % instance_ids)
 
-        (changed, instance_dict_array, new_instance_ids) = startstop_instances(module, ec2, instance_ids)
+        (changed, instance_dict_array, new_instance_ids) = startstop_instances(module, ec2, instance_ids, state)
 
-    elif module.params.get('state') == 'present':
+    elif state == 'present':
         # Changed is always set to true when provisioning new instances
         if not module.params.get('image'):
             module.fail_json(msg='image parameter is required for new instance')

library/cloud/ec2_group

@@ -230,6 +230,10 @@ def main():
                 # to 0.0.0.0/0 is added automatically but it's not
                 # reflected in the object returned by the AWS API
                 # call. We re-read the group for getting an updated object
+                # amazon sometimes takes a couple seconds to update the security group so wait till it exists
+                while len(ec2.get_all_security_groups(filters={ 'group_id': group.id, })) == 0:
+                    time.sleep(0.1)
+
                 group = ec2.get_all_security_groups(group_ids=(group.id,))[0]
             changed = True
     else:
@@ -317,7 +321,8 @@ def main():
             # when using a vpc, but no egress rules are specified, 
             # we add in a default allow all out rule, which was the
             # default behavior before egress rules were added
-            if 'out--1-None-None-None-0.0.0.0/0' not in groupRules:
+            default_egress_rule = 'out--1-None-None-None-0.0.0.0/0'
+            if default_egress_rule not in groupRules:
                 ec2.authorize_security_group_egress(
                     group_id=group.id,
                     ip_protocol=-1,
@@ -327,6 +332,9 @@ def main():
                     cidr_ip='0.0.0.0/0'
                 )
                 changed = True
+            else:
+                # make sure the default egress rule is not removed
+                del groupRules[default_egress_rule]
 
         # Finally, remove anything left in the groupRules -- these will be defunct rules
         for rule in groupRules.itervalues():

library/cloud/rds

@@ -370,6 +370,9 @@ def main():
         else:
             return 'vpc_security_groups'
 
+    # Package up the optional parameters
+    params = {}
+
     # Validate parameters for each command
     if command == 'create':
         required_vars = [ 'instance_name', 'db_engine', 'size', 'instance_type', 'username', 'password' ]
@@ -413,9 +416,6 @@ def main():
         if module.params.get(v):
             module.fail_json(msg = str("Parameter %s invalid for %s command" % (v, command)))
 
-    # Package up the optional parameters
-    params = {}
-
     if db_engine:
         params["engine"] = db_engine
 

library/cloud/route53

@@ -223,7 +223,7 @@ def main():
         # Due to a bug in either AWS or Boto, "special" characters are returned as octals, preventing round
         # tripping of things like * and @.
         decoded_name = rset.name.replace(r'\052', '*')
-        decoded_name = rset.name.replace(r'\100', '@')
+        decoded_name = decoded_name.replace(r'\100', '@')
 
         if rset.type == type_in and decoded_name == record_in:
             found_record = True

library/cloud/vsphere_guest

@@ -976,7 +976,15 @@ class DefaultVMConfig(object):
                     for k, v in value.items():
                         if k in self.check_dict[key]:
                             if not isinstance(self.check_dict[key][k], v):
-                                self.recursive_missing.append((k, v))
+                                try:
+                                    if v == int:
+                                        self.check_dict[key][k] = int(self.check_dict[key][k])
+                                    elif v == basestring:
+                                        self.check_dict[key][k] = str(self.check_dict[key][k])
+                                    else:
+                                        raise ValueError
+                                except ValueError:
+                                    self.recursive_missing.append((k, v))
                         else:
                             self.recursive_missing.append((k, v))
 

library/commands/command

@@ -84,6 +84,24 @@ EXAMPLES = '''
 - command: /usr/bin/make_database.sh arg1 arg2 creates=/path/to/database
 '''
 
+# This is a pretty complex regex, which functions as follows:
+#
+# 1. (^|\s)
+# ^ look for a space or the beginning of the line
+# 2. (creates|removes|chdir|executable|NO_LOG)=
+# ^ look for a valid param, followed by an '='
+# 3. (?P<quote>[\'"])?
+# ^ look for an optional quote character, which can either be
+#   a single or double quote character, and store it for later
+# 4. (.*?)
+# ^ match everything in a non-greedy manner until...
+# 5. (?(quote)(?<!\\)(?P=quote))((?<!\\)(?=\s)|$)
+# ^ a non-escaped space or a non-escaped quote of the same kind
+#   that was matched in the first 'quote' is found, or the end of
+#   the line is reached
+
+PARAM_REGEX = re.compile(r'(^|\s)(creates|removes|chdir|executable|NO_LOG)=(?P<quote>[\'"])?(.*?)(?(quote)(?<!\\)(?P=quote))((?<!\\)(?=\s)|$)')
+
 def main():
 
     # the command module is the one ansible module that does not take key=value args
@@ -176,38 +194,45 @@ class CommandModule(AnsibleModule):
         ''' read the input and return a dictionary and the arguments string '''
         args = MODULE_ARGS
         params = {}
-        params['chdir'] = None
-        params['creates'] = None
-        params['removes'] = None
-        params['shell'] = False
+        params['chdir']      = None
+        params['creates']    = None
+        params['removes']    = None
+        params['shell']      = False
         params['executable'] = None
         if "#USE_SHELL" in args:
             args = args.replace("#USE_SHELL", "")
             params['shell'] = True
 
-        r = re.compile(r'(^|\s)(creates|removes|chdir|executable|NO_LOG)=(?P<quote>[\'"])?(.*?)(?(quote)(?<!\\)(?P=quote))((?<!\\)(?=\s)|$)')
-        for m in r.finditer(args):
-            v = m.group(4).replace("\\", "")
-            if m.group(2) == "creates":
-                params['creates'] = v
-            elif m.group(2) == "removes":
-                params['removes'] = v
-            elif m.group(2) == "chdir":
-                v = os.path.expanduser(v)
-                v = os.path.abspath(v)
-                if not (os.path.exists(v) and os.path.isdir(v)):
-                    self.fail_json(rc=258, msg="cannot change to directory '%s': path does not exist" % v)
-                params['chdir'] = v
-            elif m.group(2) == "executable":
-                v = os.path.expanduser(v)
-                v = os.path.abspath(v)
-                if not (os.path.exists(v)):
-                    self.fail_json(rc=258, msg="cannot use executable '%s': file does not exist" % v)
-                params['executable'] = v
-            elif m.group(2) == "NO_LOG":
-                params['NO_LOG'] = self.boolean(v)
-        args = r.sub("", args)
-        params['args'] = args
+        # use shlex to split up the args, while being careful to preserve
+        # single quotes so they're not removed accidentally
+        lexer = shlex.shlex(args)
+        lexer.whitespace = '\t '
+        lexer.whitespace_split = True
+        items = list(lexer)
+
+        for x in items:
+            quoted = x.startswith('"') and x.endswith('"') or x.startswith("'") and x.endswith("'")
+            if '=' in x and not quoted:
+                # check to see if this is a special parameter for the command
+                k, v = x.split('=', 1)
+                # because we're not breaking out quotes in the shlex split
+                # above, the value of the k=v pair may still be quoted. If
+                # so, remove them.
+                if len(v) > 1 and (v.startswith('"') and v.endswith('"') or v.startswith("'") and v.endswith("'")):
+                    v = v[1:-1]
+                if k in ('creates', 'removes', 'chdir', 'executable', 'NO_LOG'):
+                    if k == "chdir":
+                        v = os.path.abspath(os.path.expanduser(v))
+                        if not (os.path.exists(v) and os.path.isdir(v)):
+                            self.fail_json(rc=258, msg="cannot change to directory '%s': path does not exist" % v)
+                    elif k == "executable":
+                        v = os.path.abspath(os.path.expanduser(v))
+                        if not (os.path.exists(v)):
+                            self.fail_json(rc=258, msg="cannot use executable '%s': file does not exist" % v)
+                    params[k] = v
+        # Remove any of the above k=v params from the args string
+        args = PARAM_REGEX.sub('', args)
+        params['args'] = args.strip()
         return (params, params['args'])
 
 main()

library/database/mysql_replication

@@ -240,7 +240,7 @@ def main():
             master_log_pos=dict(default=None),
             relay_log_file=dict(default=None),
             relay_log_pos=dict(default=None),
-            master_ssl=dict(default=None, choices=[0,1]),
+            master_ssl=dict(default=False, type='bool'),
             master_ssl_ca=dict(default=None),
             master_ssl_capath=dict(default=None),
             master_ssl_cert=dict(default=None),
@@ -337,7 +337,7 @@ def main():
         if relay_log_pos:
             chm.append("RELAY_LOG_POS=" + relay_log_pos)
         if master_ssl:
-            chm.append("MASTER_SSL=" + master_ssl)
+            chm.append("MASTER_SSL=1")
         if master_ssl_ca:
             chm.append("MASTER_SSL_CA='" + master_ssl_ca + "'")
         if master_ssl_capath:
@@ -347,7 +347,7 @@ def main():
         if master_ssl_key:
             chm.append("MASTER_SSL_KEY='" + master_ssl_key + "'")
         if master_ssl_cipher:
-            chm.append("MASTER_SSL_CIPTHER='" + master_ssl_cipher + "'")
+            chm.append("MASTER_SSL_CIPHER='" + master_ssl_cipher + "'")
         changemaster(cursor,chm)
         module.exit_json(changed=True)
     elif mode in "startslave":

library/files/copy

@@ -204,6 +204,8 @@ def main():
                 os.unlink(dest)
                 open(dest, 'w').close()
             if validate:
+                if "%s" not in validate:
+                    module.fail_json(msg="validate must contain %%s: %s" % (validate))
                 (rc,out,err) = module.run_command(validate % src)
                 if rc != 0:
                     module.fail_json(msg="failed to validate: rc:%s error:%s" % (rc,err))

library/files/file

@@ -182,11 +182,13 @@ def main():
 
     elif state in ['link','hard']:
 
-        absrc = src
-        if not os.path.isabs(absrc):
-            absrc = os.path.normpath('%s/%s' % (os.path.dirname(path), absrc))
+        if os.path.isdir(path) and not os.path.islink(path):
+            relpath = path
+        else:
+            relpath = os.path.dirname(path)
 
-        if not os.path.exists(absrc) and not force:
+        absrc = os.path.normpath('%s/%s' % (relpath, os.path.basename(src)))
+        if not os.path.exists(src) and not os.path.exists(absrc) and not force:
             module.fail_json(path=path, src=src, msg='src file does not exist, use "force=yes" if you really want to create the link: %s' % absrc)
 
         if state == 'hard':

library/files/lineinfile

@@ -154,6 +154,8 @@ def write_changes(module,lines,dest):
     validate = module.params.get('validate', None)
     valid = not validate
     if validate:
+        if "%s" not in validate:
+            module.fail_json(msg="validate must contain %%s: %s" % (validate))
         (rc, out, err) = module.run_command(validate % tmpfile)
         valid = rc == 0
         if rc != 0:

library/files/replace

@@ -90,6 +90,8 @@ def write_changes(module,contents,dest):
     validate = module.params.get('validate', None)
     valid = not validate
     if validate:
+        if "%s" not in validate:
+            module.fail_json(msg="validate must contain %%s: %s" % (validate))
         (rc, out, err) = module.run_command(validate % tmpfile)
         valid = rc == 0
         if rc != 0:

library/network/uri

@@ -406,7 +406,6 @@ def main():
     # Write the file out if requested
     if dest is not None:
         if resp['status'] == 304:
-            status_code = 304
             changed = False
         else:
             write_file(module, url, dest, content)

library/packaging/apt_repository

@@ -83,10 +83,11 @@ import tempfile
 try:
     import apt
     import apt_pkg
-    import aptsources.distro
-    distro = aptsources.distro.get_distro()
+    import aptsources.distro as aptsources_distro
+    distro = aptsources_distro.get_distro()
     HAVE_PYTHON_APT = True
 except ImportError:
+    distro = None
     HAVE_PYTHON_APT = False
 
 
@@ -97,11 +98,16 @@ def install_python_apt(module):
     if not module.check_mode:
         apt_get_path = module.get_bin_path('apt-get')
         if apt_get_path:
-            rc, so, se = module.run_command('%s update && %s install python-apt -y -q' % (apt_get_path, apt_get_path))
+            rc, so, se = module.run_command('%s update && %s install python-apt -y -q' % (apt_get_path, apt_get_path), use_unsafe_shell=True)
             if rc == 0:
-                global apt, apt_pkg
+                global apt, apt_pkg, aptsources_distro, distro, HAVE_PYTHON_APT
                 import apt
                 import apt_pkg
+                import aptsources.distro as aptsources_distro
+                distro = aptsources_distro.get_distro()
+                HAVE_PYTHON_APT = True
+            else:
+                module.fail_json(msg="Failed to auto-install python-apt. Error was: '%s'" % se.strip())
 
 class InvalidSource(Exception):
     pass
@@ -314,6 +320,8 @@ class UbuntuSourcesList(SourcesList):
 
         headers = dict(Accept='application/json')
         response, info = fetch_url(self.module, lp_api, headers=headers)
+        if info['status'] != 200:
+            self.module.fail_json(msg="failed to fetch PPA information, error was: %s" % info['msg'])
         return json.load(response)
 
     def _expand_ppa(self, path):
@@ -382,14 +390,16 @@ def main():
     update_cache = module.params['update_cache']
     sourceslist = None
 
-    if isinstance(distro, aptsources.distro.UbuntuDistribution):
-        sourceslist = UbuntuSourcesList(module,
-                                        add_ppa_signing_keys_callback=get_add_ppa_signing_key_callback(module))
-    elif isinstance(distro, aptsources.distro.DebianDistribution) or \
-            isinstance(distro, aptsources.distro.Distribution):
-        sourceslist = SourcesList()
+    if HAVE_PYTHON_APT:
+        if isinstance(distro, aptsources_distro.UbuntuDistribution):
+            sourceslist = UbuntuSourcesList(module,
+                add_ppa_signing_keys_callback=get_add_ppa_signing_key_callback(module))
+        elif HAVE_PYTHON_APT and \
+            isinstance(distro, aptsources_distro.DebianDistribution) or isinstance(distro, aptsources_distro.Distribution):
+            sourceslist = SourcesList()
     else:
-        module.fail_json(msg='Module apt_repository supports only Debian and Ubuntu.')
+        module.fail_json(msg='Module apt_repository supports only Debian and Ubuntu. ' + \
+                             'You may be seeing this because python-apt is not installed, but you requested that it not be auto-installed')
 
     sources_before = sourceslist.dump()
 

library/packaging/svr4pkg

@@ -152,7 +152,7 @@ def package_install(module, name, src, proxy, response_file, zone, category):
     os.unlink(adminfile)
     return (rc, out, err)
 
-def package_uninstall(module, name, src):
+def package_uninstall(module, name, src, category):
     adminfile = create_admin_file()
     if category:
         cmd = [ 'pkgrm', '-na', adminfile, '-Y', name ]
@@ -209,10 +209,18 @@ def main():
             (rc, out, err) = package_uninstall(module, name, src, category)
             out = out[:75]
 
-    if rc is None:
-        result['changed'] = False
-    else:
+    # Success, Warning, Interruption, Reboot all, Reboot this return codes
+    if rc in (0, 2, 3, 10, 20):
         result['changed'] = True
+    # no install nor uninstall, or failed
+    else:
+        result['changed'] = False
+
+    # Fatal error, Administration, Administration Interaction return codes
+    if rc in (1, 4 , 5):
+        result['failed'] = True
+    else:
+        result['failed'] = False
 
     if out:
         result['stdout'] = out

library/source_control/git

@@ -181,7 +181,15 @@ def get_submodule_update_params(module, git_path, cwd):
 
 def write_ssh_wrapper():
     module_dir = get_module_path()
-    fd, wrapper_path = tempfile.mkstemp(prefix=module_dir + '/')
+    try:
+        # make sure we have full permission to the module_dir, which
+        # may not be the case if we're sudo'ing to a non-root user
+        if os.access(module_dir, os.W_OK|os.R_OK|os.X_OK):
+            fd, wrapper_path = tempfile.mkstemp(prefix=module_dir + '/')
+        else:
+            raise OSError
+    except (IOError, OSError):
+        fd, wrapper_path = tempfile.mkstemp()
     fh = os.fdopen(fd, 'w+b')
     template = """#!/bin/sh
 if [ -z "$GIT_SSH_OPTS" ]; then
@@ -502,6 +510,7 @@ def main():
     if key_file or ssh_opts:
         ssh_wrapper = write_ssh_wrapper()
         set_git_ssh(ssh_wrapper, key_file, ssh_opts)
+        module.add_cleanup_file(path=ssh_wrapper)
 
     # add the git repo's hostkey 
     if module.params['ssh_opts'] is not None:

library/system/alternatives

@@ -53,7 +53,6 @@ EXAMPLES = '''
   alternatives: name=hadoop-conf link=/etc/hadoop/conf path=/etc/hadoop/conf.ansible
 '''
 
-UPDATE_ALTERNATIVES = '/usr/sbin/update-alternatives'
 DEFAULT_LINK_PRIORITY = 50
 
 def main():
@@ -71,6 +70,8 @@ def main():
     path = params['path']
     link = params['link']
 
+    UPDATE_ALTERNATIVES =  module.get_bin_path('update-alternatives',True)
+
     current_path = None
     all_alternatives = []
 

library/utilities/accelerate

@@ -479,7 +479,7 @@ class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
         if executable:
             use_unsafe_shell = True
 
-        rc, stdout, stderr = self.server.module.run_command(data['cmd'], executable=executable, use_unsafe_shell=use_unsafe_shell)
+        rc, stdout, stderr = self.server.module.run_command(data['cmd'], executable=executable, use_unsafe_shell=use_unsafe_shell, close_fds=True)
         if stdout is None:
             stdout = ''
         if stderr is None:

library/web_infrastructure/apache2_module

@@ -1,7 +1,7 @@
 #!/usr/bin/python
 #coding: utf-8 -*-
 
-# (c) 2013, Christian Berendt <berendt@b1-systems.de>
+# (c) 2013-2014, Christian Berendt <berendt@b1-systems.de>
 #
 # This module is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -44,39 +44,31 @@ EXAMPLES = '''
 - apache2_module: state=absent name=wsgi
 '''
 
-def _module_is_enabled(module):
-    name = module.params['name']
-    a2enmod_binary = module.get_bin_path("a2enmod")
-    result, stdout, stderr = module.run_command("%s -q %s" % (a2enmod_binary, name))
-    return result == 0
-
-def _module_is_disabled(module):
-    return _module_is_enabled(module) == False
+import re
 
 def _disable_module(module):
     name = module.params['name']
+    a2dismod_binary = module.get_bin_path("a2dismod")
+    result, stdout, stderr = module.run_command("%s %s" % (a2dismod_binary, name))
 
-    if _module_is_disabled(module):
+    if re.match(r'.*already disabled.*', stdout):
         module.exit_json(changed = False, result = "Success")
-
-    result, stdout, stderr  = module.run_command("a2dismod %s" % name)
-    if result != 0: 
+    elif result != 0: 
         module.fail_json(msg="Failed to disable module %s: %s" % (name, stdout))
-
-    module.exit_json(changed = True, result = "Disabled")
+    else:
+        module.exit_json(changed = True, result = "Disabled")
 
 def _enable_module(module):
     name = module.params['name']
-
-    if _module_is_enabled(module):
-        module.exit_json(changed = False, result = "Success")
-
     a2enmod_binary = module.get_bin_path("a2enmod")
     result, stdout, stderr = module.run_command("%s %s" % (a2enmod_binary, name))
-    if result != 0: 
-        module.fail_json(msg="Failed to enable module %s: %s" % (name, stdout))
 
-    module.exit_json(changed = True, result = "Enabled")
+    if re.match(r'.*already enabled.*', stdout):
+        module.exit_json(changed = False, result = "Success")
+    elif result != 0: 
+        module.fail_json(msg="Failed to enable module %s: %s" % (name, stdout))
+    else:
+        module.exit_json(changed = True, result = "Enabled")
 
 def main():
     module = AnsibleModule(
@@ -95,4 +87,3 @@ def main():
 # import module snippets
 from ansible.module_utils.basic import *
 main()
-

packaging/debian/changelog

@@ -1,32 +1,92 @@
+ansible (1.6.10) unstable; urgency=low
+
+  * 1.6.10 release
+
+ -- Michael DeHaan <michael@ansible.com>  Fri, 25 Jul 2014 13:00:00 -0500
+
+ansible (1.6.9) unstable; urgency=low
+
+  * 1.6.9 release
+
+ -- Michael DeHaan <michael@ansible.com>  Thu, 24 Jul 2014 23:30:00 -0500
+
+ansible (1.6.8) unstable; urgency=low
+
+  * 1.6.8 release
+
+ -- Michael DeHaan <michael@ansible.com>  Tue, 22 Jul 2014 17:30:00 -0500
+
+ansible (1.6.7) unstable; urgency=low
+
+  * 1.6.7 release
+
+ -- Michael DeHaan <michael@ansible.com>  Mon, 21 Jul 2014 12:30:00 -0500
+
+ansible (1.6.6) unstable; urgency=low
+
+  * 1.6.6 release
+
+ -- Michael DeHaan <michael@ansible.com>  Tue, 01 Jul 2014 15:30:00 -0500
+
+ansible (1.6.5) unstable; urgency=low
+
+  * 1.6.5 release
+
+ -- Michael DeHaan <michael@ansible.com>  Wed, 25 Jun 2014 22:30:00 -0500
+
+ansible (1.6.4) unstable; urgency=low
+
+  * 1.6.4 release
+
+ -- Michael DeHaan <michael@ansible.com>  Wed, 25 Jun 2014 17:30:00 -0500
+
+ansible (1.6.3) unstable; urgency=low
+
+  * 1.6.3 release
+
+ -- Michael DeHaan <michael@ansible.com>  Mon, 09 Jun 2014 16:30:00 -0500
+
+ansible (1.6.2) unstable; urgency=low
+
+  * 1.6.2 release
+
+ -- Michael DeHaan <michael@ansible.com>  Fri, 23 May 2014 17:30:00 -0500
+
+ansible (1.6.1) unstable; urgency=low
+
+  * 1.6.1 release
+
+ -- Michael DeHaan <michael@ansible.com>  Wed, 07 May 2014 13:30:00 -0500
+
 ansible (1.6) unstable; urgency=low
 
   * 1.6.0 release
- 
- -- Michael DeHaan <michael.dehaan@gmail.com>  Mon, 05 May 2014 17:30:03 -0500
+
+ -- Michael DeHaan <michael@ansible.com>  Mon, 05 May 2014 17:30:03 -0500
 
 ansible (1.5.5) unstable; urgency=low
 
   * 1.5.5 release
 
-  -- Michael DeHaan <michael@ansible.com>  Fri, 18 April 2014 15:00:00 -0500
+ -- Michael DeHaan <michael@ansible.com>  Fri, 18 April 2014 15:00:00 -0500
 
 ansible (1.5.4) unstable; urgency=low
 
   * 1.5.4 release
 
-  -- Michael DeHaan <michael@ansible.com>  Tue, 01 April 2014 08:46:00 -0500
+ -- Michael DeHaan <michael@ansible.com>  Tue, 01 April 2014 08:46:00 -0500
 
 ansible (1.5.3) unstable; urgency=low
 
   * 1.5.3 release
 
-  -- Michael DeHaan <michael@ansible.com>  Thu, 13 March 2014 08:46:00 -0500
+ -- Michael DeHaan <michael@ansible.com>  Thu, 13 March 2014 08:46:00 -0500
 
 ansible (1.5.2) unstable; urgency=low
 
   * 1.5.2 release
 
-  -- Michael DeHaan <michael@ansible.com>  Tue, 11 March 2014 08:46:00 -0500
+ -- Michael DeHaan <michael@ansible.com>  Tue, 11 March 2014 08:46:00 -0500
 
 ansible (1.5.1) unstable; urgency=low
 
@@ -37,7 +97,7 @@ ansible (1.5.1) unstable; urgency=low
 ansible (1.5) unstable; urgency=low
 
   * 1.5 release
- 
+
  -- Michael DeHaan <michael@ansible.com>  Fri, 28 February 2014 00:00:00 -0500
 
 ansible (1.4.5) unstable; urgency=low
@@ -72,8 +132,8 @@ ansible (1.4.1) unstable; urgency=low
 
 ansible (1.4) unstable; urgency=low
 
-  * 1.4 release 
- 
+  * 1.4 release
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Thu, 21 November 2013 15:00:01 -0500
 
 ansible (1.3) unstable; urgency=low
@@ -83,15 +143,15 @@ ansible (1.3) unstable; urgency=low
  -- Michael DeHaan <michael.dehaan@gmail.com>  Fri, 13 September 2013 18:00:00 -0500
 
 ansible (1.2.2) unstable; urgency=low
-  
+
   * 1.2.2 release
- 
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Thu, 04 July 2013 21:45:01 -0500
 
 ansible (1.2.1) unstable; urgency=low
 
   * 1.2.1 release
- 
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Thu, 04 July 2013 21:45:00 -0500
 
 ansible (1.2) unstable; urgency=low
@@ -107,33 +167,33 @@ ansible (1.1) unstable; urgency=low
  -- Michael DeHaan <michael.dehaan@gmail.com>  Tue, 02 Mar 2013 20:38:00 -0500
 
 ansible (1.0) unstable; urgency=low
-  
-  * 1.0 release 
- 
+
+  * 1.0 release
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Fri, 01 Feb 2013 21:34:34 -0500
 
 ansible (0.9) unstable; urgency=low
-  
-  * 0.9 release 
- 
+
+  * 0.9 release
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Fri, 30 Nov 2012 22:00:00 -0400
 
 ansible (0.8) unstable; urgency=low
-  
+
   * 0.8 update
- 
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Fri, 19 Oct 2012 20:50:00 -0400
 
 ansible (0.7) unstable; urgency=low
-  
+
   * 0.7 update
- 
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Thu, 06 Aug 2012 18:50:00 -0400
 
 ansible (0.6) unstable; urgency=low
-  
+
   * 0.6 update
- 
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Mon, 06 Aug 2012 19:50:00 -0400
 
 ansible (0.5) unstable; urgency=low
@@ -143,9 +203,9 @@ ansible (0.5) unstable; urgency=low
  -- Michael DeHaan <michael.dehaan@gmail.com>  Wed, 04 Jul 2012 13:40:00 -0400
 
 ansible (0.4) unstable; urgency=low
-  
+
   * 0.4 update
- 
+
  -- Michael DeHaan <michael.dehaan@gmail.com>  Wed, 23 May 2012 19:40:00 -0400
 
 ansible (0.3) unstable; urgency=low

packaging/debian/control

@@ -1,13 +1,16 @@
 Source: ansible
 Section: admin
 Priority: optional
-Maintainer: Henry Graham (hzgraham) <Henry.Graham@mail.wvu.edu>
-Build-Depends: cdbs, debhelper (>= 5.0.0), asciidoc
-Standards-Version: 3.9.1
+Standards-Version: 3.9.3
+Maintainer: Michael DeHaan <michael.dehaan@gmail.com>
+Build-Depends: cdbs, debhelper (>= 5.0.0), asciidoc, python, python-support, python-setuptools
 Homepage: http://ansible.github.com/
 
 Package: ansible
 Architecture: all
-Depends: python, python-support (>= 0.90), python-jinja2, python-yaml, python-paramiko, python-httplib2, sshpass
-Description: Ansible Application
- Ansible is a extra-simple tool/API for doing 'parallel remote things' over SSH executing commands, running "modules", or executing larger 'playbooks' that can serve as a configuration management or deployment system.
+Depends: python, python-support (>= 0.90), python-jinja2, python-yaml, python-paramiko, python-httplib2, sshpass, ${misc:Depends}
+Description: A radically simple IT automation platform
+ A radically simple IT automation platform that makes your applications and
+ systems easier to deploy. Avoid writing scripts or custom code to deploy and
+ update your applications— automate in a language that approaches plain English,
+ using SSH, with no agents to install on remote systems.

packaging/rpm/ansible.spec

@@ -1,11 +1,12 @@
 %define name ansible
+%define ansible_version $VERSION
 
 %if 0%{?rhel} == 5
 %define __python /usr/bin/python26
 %endif
 
 Name:      %{name}
-Version:   1.6
+Version:   %{ansible_version}
 Release:   1%{?dist}
 Url:       http://www.ansible.com
 Summary:   SSH-based application deployment, configuration management, and IT orchestration platform
@@ -114,6 +115,36 @@ rm -rf %{buildroot}
 
 %changelog
 
+* Fri Jul 25 2014 Michael DeHaan <michael@ansible.com> - 1.6.10
+- Release 1.6.10
+
+* Thu Jul 24 2014 Michael DeHaan <michael@ansible.com> - 1.6.9
+- Release 1.6.9
+
+* Tue Jul 22 2014 Michael DeHaan <michael@ansible.com> - 1.6.8
+- Release 1.6.8
+
+* Mon Jul 21 2014 Michael DeHaan <michael@ansible.com> - 1.6.7
+- Release 1.6.7
+
+* Tue Jul 01 2014 Michael DeHaan <michael@ansible.com> - 1.6.6
+- Release 1.6.6
+
+* Wed Jun 25 2014 Michael DeHaan <michael@ansible.com> - 1.6.5
+- Release 1.6.5
+
+* Wed Jun 25 2014 Michael DeHaan <michael@ansible.com> - 1.6.4
+- Release 1.6.4
+
+* Mon Jun 09 2014 Michael DeHaan <michael@ansible.com> - 1.6.3
+- Release 1.6.3
+
+* Fri May 23 2014 Michael DeHaan <michael@ansible.com> - 1.6.2
+- Release 1.6.2
+
+* Wed May 07 2014 Michael DeHaan <michael@ansible.com> - 1.6.1
+- Release 1.6.1
+
 * Mon May 05 2014 Michael DeHaan <michael@ansible.com> - 1.6.0
 - Release 1.6.0
 

test/integration/Makefile

@@ -14,7 +14,17 @@ else
 CREDENTIALS_ARG =
 endif
 
-all: non_destructive destructive check_mode test_hash test_handlers
+all: non_destructive destructive check_mode test_hash test_handlers test_group_by parsing
+
+parsing:
+	ansible-playbook bad_parsing.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -vvv $(TEST_FLAGS) --tags common,scenario1; [ $$? -eq 3 ]
+	ansible-playbook bad_parsing.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -vvv $(TEST_FLAGS) --tags common,scenario2; [ $$? -eq 3 ]
+	ansible-playbook bad_parsing.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -vvv $(TEST_FLAGS) --tags common,scenario3; [ $$? -eq 3 ]
+	ansible-playbook bad_parsing.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -vvv $(TEST_FLAGS) --tags common,scenario4; [ $$? -eq 3 ]
+	ansible-playbook good_parsing.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+
+unicode:
+	ansible-playbook unicode.yml -i $(INVENTORY) -e @$(VARS_FILE) -v $(TEST_FLAGS)
 
 non_destructive:
 	ansible-playbook non_destructive.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
@@ -25,6 +35,9 @@ destructive:
 check_mode:
 	ansible-playbook check_mode.yml -i $(INVENTORY) -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v --check $(TEST_FLAGS)
 
+test_group_by:
+	ansible-playbook test_group_by.yml -i inventory.group_by -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
+
 test_handlers:
 	ansible-playbook test_handlers.yml -i inventory.handlers -e @$(VARS_FILE) $(CREDENTIALS_ARG) -v $(TEST_FLAGS)
 

test/integration/all.yml

@@ -3,4 +3,4 @@
 - include: destructive.yml
 - include: rackspace.yml
 - include: amazon.yml
-
+- include: good_parsing.yml

test/integration/bad_parsing.yml

@@ -0,0 +1,12 @@
+- hosts: testhost
+
+  # the following commands should all parse fine and execute fine
+  # and represent quoting scenarios that should be legit
+
+  gather_facts: False
+
+  roles: 
+
+    # this one has a lot of things that should fail, see makefile for operation w/ tags
+
+    - { role: test_bad_parsing } 

test/integration/good_parsing.yml

@@ -0,0 +1,9 @@
+- hosts: testhost
+
+  # the following commands should all parse fine and execute fine
+  # and represent quoting scenarios that should be legit
+
+  gather_facts: False
+
+  roles: 
+    - { role: test_good_parsing, tags: test_good_parsing }

test/integration/inventory.group_by

@@ -0,0 +1,6 @@
+[lamini]
+alpaca genus=vicugna
+llama genus=lama
+
+[lamini:vars]
+ansible_connection=local

test/integration/roles/setup_ec2/tasks/main.yml

@@ -17,7 +17,7 @@
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
 - name: generate random string
-  shell: python -c "import string,random; print ''.join(random.choice(string.ascii_lowercase) for _ in xrange(8));"
+  command: python -c "import string,random; print ''.join(random.choice(string.ascii_lowercase) for _ in xrange(8));"
   register: random_string
   tags:
     - prepare

test/integration/roles/test_apt/tasks/apt.yml

@@ -63,7 +63,6 @@
     that:
         - "'invocation' in apt_result"
         - "'changed' in apt_result"
-        - "'item' in apt_result"
         - "'stderr' in apt_result"
         - "'stdout' in apt_result"
         - "'stdout_lines' in apt_result"

test/integration/roles/test_async/tasks/main.yml

@@ -35,7 +35,6 @@
         - "'end' in async_result"
         - "'finished' in async_result"
         - "'invocation' in async_result"
-        - "'item' in async_result"
         - "'rc' in async_result"
         - "'start' in async_result"
         - "'stderr' in async_result"

test/integration/roles/test_bad_parsing/tasks/main.yml

@@ -0,0 +1,53 @@
+# test code for the ping module
+# (c) 2014, Michael DeHaan <michael@ansible.com>
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# the following tests all raise errors, to use them in a Makefile, we run them with different flags, as
+# otherwise ansible stops at the first one and we want to ensure STOP conditions for each
+
+- set_fact: 
+    test_file: "{{ output_dir }}/ansible_test_file"  # FIXME, use set tempdir
+    test_input: "owner=test" 
+    bad_var: "{{ output_dir }}' owner=test"
+    chdir: "mom chdir=/tmp"
+  tags: common
+
+- file: name={{test_file}} state=touch
+  tags: common
+
+- name: test that we cannot insert arguments
+  file: path={{ test_file }} {{ test_input }}
+  failed_when: False # ignore the module, just test the parser
+  tags: scenario1
+
+- name: test that we cannot duplicate arguments
+  file: path={{ test_file }} owner=test2 {{ test_input }}
+  failed_when: False # ignore the module, just test the parser
+  tags: scenario2
+
+- name: test that we can't do this for the shell module
+  shell: echo hi {{ chdir }}
+  failed_when: False
+  tags: scenario3
+
+- name: test that we can't go all Little Bobby Droptables on a quoted var to add more
+  file: "name={{ bad_var }}"
+  failed_when: False
+  tags: scenario4
+
+
+

test/integration/roles/test_command_shell/tasks/main.yml

@@ -167,4 +167,4 @@
 - name: assert that the file was removed by the shell
   assert:
     that:
-      - "shell_result4.changed == False"
+      - "shell_result4.changed == False"

test/integration/roles/test_copy/tasks/main.yml

@@ -151,5 +151,11 @@
 #  ignore_errors: True
 #  register: failed_copy
 
+- name: copy already copied directory again
+  copy: src=subdir dest={{output_subdir | expanduser}} owner={{ansible_ssh_user}}
+  register: copy_result5
 
-
+- name: assert that the directory was not changed
+  assert:
+    that:
+      - "not copy_result5|changed"

test/integration/roles/test_ec2_group/tasks/main.yml

@@ -191,7 +191,6 @@
     that:
        - 'result.changed'
        - 'result.group_id.startswith("sg-")'
-       - 'result.item == ""'
 
 # ============================================================
 - name: test state=present (expected changed=false)
@@ -209,7 +208,6 @@
     that:
        - 'not result.changed'
        - 'result.group_id.startswith("sg-")'
-       - 'result.item == ""'
 
 # ============================================================
 - name: test state=absent (expected changed=true)
@@ -228,7 +226,6 @@
     that:
        - 'result.changed'
        - 'not result.group_id'
-       - 'result.item == ""'
 
 # ============================================================
 - name: test state=absent (expected changed=false)
@@ -247,4 +244,3 @@
     that:
        - 'not result.changed'
        - 'not result.group_id'
-       - 'result.item == ""'

test/integration/roles/test_good_parsing/tasks/main.yml

@@ -0,0 +1,118 @@
+# test code for the ping module
+# (c) 2014, Michael DeHaan <michael@ansible.com>
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+# various tests of things that should not cause parsing problems
+
+- set_fact: 
+    test_input: "a=1 a=2 a=3"
+
+- set_fact:
+    multi_line: |
+      echo old
+      echo mcdonald
+      echo had
+      echo a 
+      echo farm
+
+- shell: echo "dog"
+  register: result
+
+- assert:
+    that:
+       result.cmd == 'echo "dog"'
+
+- shell: echo 'dog'
+  register: result
+
+- assert:
+    that:
+       result.cmd == 'echo \'dog\''
+
+- name: a quoted argument is not sent to the shell module as anything but a string parameter
+  shell: echo 'dog' 'executable=/usr/bin/python'
+  register: result
+ 
+- debug: var=result.cmd
+ 
+- assert:
+    that:
+       result.cmd == "echo 'dog' 'executable=/usr/bin/python'"
+
+- name: it is valid to pass multiple key=value arguments because the shell doesn't check key=value arguments
+  shell: echo quackquack=here quackquack=everywhere
+  register: result
+ 
+- assert:
+    that:
+      result.cmd == 'echo quackquack=here quackquack=everywhere'
+
+- name: the same is true with quoting
+  shell: echo "quackquack=here quackquack=everywhere"
+  register: result
+     
+- assert:
+    that:
+      result.cmd == 'echo "quackquack=here quackquack=everywhere"'
+
+- name: the same is true with quoting (B)
+  shell: echo "quackquack=here" "quackquack=everywhere"
+  register: result
+
+- name: the same is true with quoting (C)
+  shell: echo "quackquack=here" 'quackquack=everywhere'
+  register: result
+
+- name: the same is true with quoting (D)
+  shell: echo "quackquack=here" 'quackquack=everywhere'
+  register: result
+
+- name: the same is true with quoting (E)
+  shell: echo {{ test_input }}
+  register: result
+
+- assert: 
+     that:
+        result.cmd == "echo a=1 a=2 a=3" 
+
+- name: more shell duplicates
+  shell: echo foo=bar foo=bar
+  register: result
+
+- assert:
+    that:
+       result.cmd == "echo foo=bar foo=bar"
+
+- name: multi-line inline shell commands (should use script module but hey) are a thing
+  shell: "{{ multi_line }}"
+  register: result
+
+- debug: var=result
+
+- assert:
+    that:
+       result.stdout_lines == [ 'old', 'mcdonald', 'had', 'a', 'farm' ]
+
+- name: passing same arg to shell command is legit
+  shell: echo foo --arg=a --arg=b
+  failed_when: False # just catch the exit code, parse error is what I care about, but should register and compare result
+  register: result
+
+- assert:
+    that:
+       # command shouldn't end in spaces, amend test once fixed
+       - result.cmd == "echo foo --arg=a --arg=b"

test/integration/roles/test_handlers/meta/main.yml

@@ -1,3 +1,2 @@
-dependencies: 
-  - prepare_tests
+dependencies: []
 

test/integration/roles/test_handlers/tasks/main.yml

@@ -16,8 +16,23 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
-- name: notify the handler
+
+- name: reset handler_called variable to false for all hosts
+  set_fact:
+    handler_called: False
+
+- name: notify the handler for host A only
   shell: echo
   notify: 
     - set handler fact
+  when: inventory_hostname == 'A'
 
+- name: force handler execution now
+  meta: "flush_handlers"
+
+- debug: var=handler_called
+
+- name: validate the handler only ran on one host
+  assert:
+    that:
+    - "inventory_hostname == 'A' and handler_called == True or handler_called == False"

test/integration/roles/test_iterators/tasks/main.yml

@@ -19,7 +19,7 @@
 # WITH_ITEMS
 
 - name: test with_items
-  set_fact: "{{ item + '=moo' }}"
+  set_fact: "{{ item }}=moo"
   with_items:
     - 'foo'
     - 'bar'
@@ -36,7 +36,7 @@
 # WITH_NESTED
 
 - name: test with_nested
-  set_fact: "{{ item.0 + item.1 + '=x' }}"
+  set_fact: "{{ item.0 + item.1 }}=x"
   with_nested:
     - [ 'a', 'b' ]
     - [ 'c', 'd' ]        
@@ -57,7 +57,7 @@
 # WITH_SEQUENCE
 
 - name: test with_sequence
-  set_fact: "{{ 'x' + item + '=' + item }}"
+  set_fact: "{{ 'x' + item }}={{ item }}"
   with_sequence: start=0 end=3
 
 - name: verify with_sequence
@@ -71,7 +71,7 @@
 # WITH_RANDOM_CHOICE
 
 - name: test with_random_choice
-  set_fact: "{{ 'random=' + item }}"
+  set_fact: "random={{ item }}"
   with_random_choice:
     - "foo"
     - "bar" 
@@ -84,7 +84,7 @@
 # WITH_SUBELEMENTS
 
 - name: test with_subelements
-  set_fact: "{{ '_'+ item.0.id + item.1 + '=' + item.1 }}"
+  set_fact: "{{ '_'+ item.0.id + item.1 }}={{ item.1 }}"
   with_subelements:
     - element_data
     - the_list
@@ -101,7 +101,7 @@
 
 - name: test with_together
   #shell: echo {{ item }}
-  set_fact: "{{ item.0 + '=' + item.1 }}"
+  set_fact: "{{ item.0 }}={{ item.1 }}"
   with_together:
     - [ 'a', 'b', 'c', 'd' ]
     - [ '1', '2', '3', '4' ]
@@ -124,7 +124,7 @@
 
 - name: test with_first_found
   #shell: echo {{ item }}
-  set_fact: "{{ 'first_found=' + item }}"
+  set_fact: "first_found={{ item }}"
   with_first_found:
     - "{{ output_dir + '/does_not_exist' }}"
     - "{{ output_dir + '/foo1' }}"
@@ -146,7 +146,7 @@
 
 - name: test with_lines
   #shell: echo "{{ item }}"
-  set_fact: "{{ item + '=set' }}" 
+  set_fact: "{{ item }}=set" 
   with_lines: for i in $(seq 1 5); do echo "l$i" ; done;
 
 - name: verify with_lines results
@@ -164,7 +164,7 @@
   register: list_data
 
 - name: create indexed list
-  set_fact: "{{ item[1] + item[0]|string + '=set' }}"
+  set_fact: "{{ item[1] + item[0]|string }}=set"
   with_indexed_items: list_data.stdout_lines  
 
 - name: verify with_indexed_items result
@@ -179,8 +179,7 @@
 # WITH_FLATTENED
 
 - name: test with_flattened
-  #shell: echo {{ item + "test" }}
-  set_fact: "{{ item + '=flattened' }}"
+  set_fact: "{{ item }}=flattened"
   with_flattened:
     - [ 'a__' ]
     - [ 'b__', ['c__', 'd__'] ]        

test/integration/roles/test_lookups/tasks/main.yml

@@ -92,7 +92,7 @@
 
 # https://github.com/ansible/ansible/issues/6550
 - name: confirm pipe lookup works with multiple positional args
-  debug: msg="{{ lookup('pipe', 'ls /tmp /') }}"         
+  debug: msg="{{ lookup('pipe', 'ls -l /tmp') }}"         
 
 
 

test/integration/roles/test_yum/tasks/yum.yml

@@ -69,7 +69,6 @@
     that:
         - "'invocation' in yum_result"
         - "'changed' in yum_result"
-        - "'item' in yum_result"
         - "'msg' in yum_result"
         - "'rc' in yum_result"
         - "'results' in yum_result"

test/integration/test_group_by.yml

@@ -0,0 +1,111 @@
+# test code for the group_by module
+# (c) 2014, Chris Church <cchurch@ansible.com>
+
+# This file is part of Ansible
+#
+# Ansible is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Ansible is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
+
+- hosts: lamini
+  gather_facts: false
+  tasks:
+    - name: group by genus
+      group_by: key={{ genus }}
+    - name: group by first three letters of genus with key in quotes
+      group_by: key="{{ genus | truncate(3, true, '') }}"
+    - name: group by first two letters of genus with key not in quotes
+      group_by: key={{ genus | truncate(2, true, '') }}
+    - name: group by genus in uppercase using complex args
+      group_by: { key: "{{ genus | upper() }}" }
+
+- hosts: vicugna
+  gather_facts: false
+  tasks:
+    - name: verify that only the alpaca is in this group
+      assert: { that: "inventory_hostname == 'alpaca'" }
+    - name: set a fact to check that we ran this play
+      set_fact: genus_vicugna=true
+
+- hosts: lama
+  gather_facts: false
+  tasks:
+    - name: verify that only the llama is in this group
+      assert: { that: "inventory_hostname == 'llama'" }
+    - name: set a fact to check that we ran this play
+      set_fact: genus_lama=true
+
+- hosts: vic
+  gather_facts: false
+  tasks:
+    - name: verify that only the alpaca is in this group
+      assert: { that: "inventory_hostname == 'alpaca'" }
+    - name: set a fact to check that we ran this play
+      set_fact: genus_vic=true
+
+- hosts: lam
+  gather_facts: false
+  tasks:
+    - name: verify that only the llama is in this group
+      assert: { that: "inventory_hostname == 'llama'" }
+    - name: set a fact to check that we ran this play
+      set_fact: genus_lam=true
+
+- hosts: vi
+  gather_facts: false
+  tasks:
+    - name: verify that only the alpaca is in this group
+      assert: { that: "inventory_hostname == 'alpaca'" }
+    - name: set a fact to check that we ran this play
+      set_fact: genus_vi=true
+
+- hosts: la
+  gather_facts: false
+  tasks:
+    - name: verify that only the llama is in this group
+      assert: { that: "inventory_hostname == 'llama'" }
+    - name: set a fact to check that we ran this play
+      set_fact: genus_la=true
+
+- hosts: VICUGNA
+  gather_facts: false
+  tasks:
+    - name: verify that only the alpaca is in this group
+      assert: { that: "inventory_hostname == 'alpaca'" }
+    - name: set a fact to check that we ran this play
+      set_fact: genus_VICUGNA=true
+
+- hosts: LAMA
+  gather_facts: false
+  tasks:
+    - name: verify that only the llama is in this group
+      assert: { that: "inventory_hostname == 'llama'" }
+    - name: set a fact to check that we ran this play
+      set_fact: genus_LAMA=true
+
+- hosts: '{{genus'
+  gather_facts: false
+  tasks:
+    - name: no hosts should match this group
+      fail: msg="should never get here"
+
+- hosts: alpaca
+  gather_facts: false
+  tasks:
+    - name: check that alpaca matched all four groups
+      assert: { that: ["genus_vicugna", "genus_vic", "genus_vi", "genus_VICUGNA"] }
+
+- hosts: llama
+  gather_facts: false
+  tasks:
+    - name: check that llama matched all four groups
+      assert: { that: ["genus_lama", "genus_lam", "genus_la", "genus_LAMA"] }

test/integration/test_handlers.yml

@@ -17,8 +17,9 @@
             - "not hostvars[inventory_hostname]['handler1_called']"
             - "'handler2_called' in hostvars[inventory_hostname]"
 
-#- hosts: testgroup
-#  gather_facts: False
-#  connection: local
-#  roles:
-#  - { role: test_handlers_meta }
+- name: test handlers
+  hosts: testgroup
+  gather_facts: False
+  connection: local
+  roles:
+  - { role: test_handlers }

test/integration/unicode.yml

@@ -0,0 +1,42 @@
+---
+- name: 'A play with unicode: ¢ £ ¤ ¥'
+  hosts: localhost
+  connection: local
+  vars:
+    test_var: 'Ī ī Ĭ ĭ Į į İ ı IJ ij Ĵ ĵ Ķ ķ ĸ Ĺ ĺ Ļ ļ Ľ ľ Ŀ ŀ Ł ł Ń ń Ņ ņ Ň ň ʼn Ŋ ŋ Ō ō Ŏ ŏ Ő ő Œ'
+    num_hosts: 5
+    hostnames:
+        - 'host-#ϬϭϮϯϰ'
+        - 'host-ͰͱͲͳʹ͵'
+        - 'host-ΙΚΛΜΝΞ'
+        - 'host-στυφχψ'
+        - 'host-ϬϭϮϯϰϱ'
+
+  tasks:
+    - name: 'A task name with unicode: è é ê ë'
+      debug: msg='hi there'
+
+    - name: 'A task with unicode parameters'
+      debug: var=test_var
+
+      # € ‚ ƒ „ … † ‡ ˆ ‰ Š ‹ Œ Ž ‘ ’ “ ” • – — ˜ ™ š › œ ž Ÿ ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ­ ®'
+
+    - name: 'A task using with_items containing unicode'
+      debug: msg='{{item}}'
+      with_items:
+        - '¯ ° ± ² ³ ´ µ ¶ · ¸ ¹ º » ¼ ½ ¾ ¿ À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö ×'
+        - 'Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ Ā'
+        - 'ā Ă ă Ą ą Ć ć Ĉ ĉ Ċ ċ Č č Ď ď Đ đ Ē ē Ĕ ĕ Ė ė Ę ę Ě ě Ĝ ĝ Ğ ğ Ġ ġ Ģ ģ Ĥ ĥ Ħ ħ Ĩ ĩ'
+
+    - add_host:
+        name: '{{hostnames}}.{{item}}'
+        groups: 'ĪīĬĭ'
+        ansible_connection: local
+        host_id: '{{item}}'
+      with_sequence: start=1 end={{num_hosts}} format=%d
+
+- name: 'A play for hosts in group: ĪīĬĭ'
+  hosts: 'ĪīĬĭ'
+  gather_facts: true
+  tasks:
+    - debug: msg='Unicode is a good thing ™'

test/units/TestUtils.py

@@ -16,6 +16,7 @@ import ansible.utils
 import ansible.errors
 import ansible.constants as C
 import ansible.utils.template as template2
+from ansible.utils.splitter import split_args
 
 from ansible import __version__
 
@@ -244,24 +245,6 @@ class TestUtils(unittest.TestCase):
         # Just a string
         self.assertEqual(ansible.utils.parse_json('foo'), dict(failed=True, parsed=False, msg='foo'))
 
-    def test_smush_braces(self):
-        self.assertEqual(ansible.utils.smush_braces('{{ foo}}'), '{{foo}}')
-        self.assertEqual(ansible.utils.smush_braces('{{foo }}'), '{{foo}}')
-        self.assertEqual(ansible.utils.smush_braces('{{ foo }}'), '{{foo}}')
-
-    def test_smush_ds(self):
-        # list
-        self.assertEqual(ansible.utils.smush_ds(['foo={{ foo }}']), ['foo={{foo}}'])
-
-        # dict
-        self.assertEqual(ansible.utils.smush_ds(dict(foo='{{ foo }}')), dict(foo='{{foo}}'))
-
-        # string
-        self.assertEqual(ansible.utils.smush_ds('foo={{ foo }}'), 'foo={{foo}}')
-
-        # int
-        self.assertEqual(ansible.utils.smush_ds(0), 0)
-
     def test_parse_yaml(self):
         #json
         self.assertEqual(ansible.utils.parse_yaml('{"foo": "bar"}'), dict(foo='bar'))
@@ -677,3 +660,59 @@ class TestUtils(unittest.TestCase):
         diff = '\n'.join(diff)
         self.assertEqual(diff, unicode(standard_expected))
 
+    def test_split_args(self):
+        # split_args is a smarter shlex.split for the needs of the way ansible uses it
+
+        def _split_info(input, desired, actual):
+            print "SENT: ", input
+            print "WANT: ", desired 
+            print "GOT: ", actual
+
+        def _test_combo(input, desired):
+            actual = split_args(input)
+            _split_info(input, desired, actual)
+            assert actual == desired
+
+        # trivial splitting
+        _test_combo('a b=c d=f',                   ['a', 'b=c', 'd=f' ])
+
+        # mixed quotes
+        _test_combo('a b=\'c\' d="e" f=\'g\'',     ['a', "b='c'", 'd="e"', "f='g'" ])
+
+        # with spaces
+        # FIXME: this fails, commenting out only for now
+        # _test_combo('a "\'one two three\'"',     ['a', "'one two three'" ])
+
+        # TODO: ...
+        # jinja2 preservation
+        _test_combo('a {{ y }} z',                   ['a', '{{ y }}', 'z' ])
+
+        # jinja2 preservation with spaces and filters and other hard things
+        _test_combo(
+            'a {{ x | filter(\'moo\', \'param\') }} z {{ chicken }} "waffles"', 
+            ['a', "{{ x | filter('moo', 'param') }}", 'z', '{{ chicken }}', '"waffles"']
+        )
+
+        # invalid quote detection
+        with self.assertRaises(Exception):
+            split_args('hey I started a quote"')
+        with self.assertRaises(Exception):
+            split_args('hey I started a\' quote')
+
+        # jinja2 loop blocks with lots of complexity
+        _test_combo(
+            # in memory of neighbors cat
+            'a {% if x %} y {%else %} {{meow}} {% endif %} cookiechip\ndone',
+            ['a', '{% if x %}', 'y', '{%else %}', '{{meow}}', '{% endif %}', 'cookiechip\ndone']
+        )
+
+        # test space preservation within quotes
+        _test_combo(
+            'content="1 2  3   4    "  foo=bar',
+            ['content="1 2  3   4    "', 'foo=bar']
+        )
+
+        # invalid jinja2 nesting detection
+        # invalid quote nesting detection
+    
+