Version bump for 1.3.3

Fixes #4321

Conflicts:
	library/utilities/accelerate

CHANGELOG.md

@@ -1,7 +1,41 @@
 Ansible Changes By Release
 ==========================
 
-1.3 "Top of the World" - Release pending!
+1.4 "Could This Be Magic" - Release pending!
+
+Highlighted new features:
+
+* TBA
+
+New modules:
+
+* TBA
+
+Misc changes:
+
+* TBA
+
+1.3.2 "Top of the World" (reprise) - September 19th, 2013
+
+* Multiple accelerate mode fixes:
+  - Make packet reception less greedy, so multiple frames of data are not consumed by one call.
+  - Adding two timeout values (one for connection and one for data reception timeout).
+  - Added keepalive packets, so async mode is no longer required for long-running tasks.
+  - Modified accelerate daemon to use the verbose logging level of the ansible command that started it.
+  - Fixed bug where accelerate would not work in check-mode.
+
+1.3.2 "Top of the World" (reprise) - September 19th, 2013
+
+* The ControlPath variable for ssh connections is now configurable in the ansible.cfg (and via the environment variable ANSIBLE_SSH_CONTROL_PATH).
+* Added a -vvvv level, which will show SSH client debugging information in the event of a failure.
+* Fixed bug in apt_repository module where the repository cache was not being updated.
+* Fixed bug where "too many open files" errors would be encountered due to pseudo TTY's not being closed properly.
+
+1.3.1 "Top of the World" (reprise) - September 16th, 2013
+
+* Fixing a bug in accelerate mode whereby the gather_facts step would always be run via sudo regardless of the play settings.
+
+1.3 "Top of the World" - September 13th, 2013
 
 Highlighted new features:
 

RELEASES.txt

@@ -1,7 +1,8 @@
 Ansible Releases at a Glance
 ============================
 
-1.3    "Top of the World" ----- Release Pending
+1.4    "Could This Be Magic?" - active development
+1.3    "Top of the World" ----- 09-13-2013
 1.2.3  "Hear About It Later" -- 08-21-2013
 1.2.2  "Hear About It Later" -- 07-05-2013
 1.2.1  "Hear About It Later" -- 07-04-2013

VERSION

@@ -1 +1 @@
-1.3
+1.3.3

docs/man/man1/ansible-doc.1

@@ -1,13 +1,22 @@
 '\" t
 .\"     Title: ansible-doc
 .\"    Author: [see the "AUTHOR" section]
-.\" Generator: DocBook XSL Stylesheets v1.75.2 <http://docbook.sf.net/>
-.\"      Date: 04/02/2013
+.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
+.\"      Date: 10/07/2013
 .\"    Manual: System administration commands
-.\"    Source: Ansible 1.2
+.\"    Source: Ansible 1.3.3
 .\"  Language: English
 .\"
-.TH "ANSIBLE\-DOC" "1" "04/02/2013" "Ansible 1\&.2" "System administration commands"
+.TH "ANSIBLE\-DOC" "1" "10/07/2013" "Ansible 1\&.3\&.3" "System administration commands"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
 .\" -----------------------------------------------------------------
 .\" * set default formatting
 .\" -----------------------------------------------------------------

docs/man/man1/ansible-playbook.1

@@ -1,13 +1,22 @@
 '\" t
 .\"     Title: ansible-playbook
 .\"    Author: [see the "AUTHOR" section]
-.\" Generator: DocBook XSL Stylesheets v1.75.2 <http://docbook.sf.net/>
-.\"      Date: 06/03/2013
+.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
+.\"      Date: 10/07/2013
 .\"    Manual: System administration commands
-.\"    Source: Ansible 1.2
+.\"    Source: Ansible 1.3.3
 .\"  Language: English
 .\"
-.TH "ANSIBLE\-PLAYBOOK" "1" "06/03/2013" "Ansible 1\&.2" "System administration commands"
+.TH "ANSIBLE\-PLAYBOOK" "1" "10/07/2013" "Ansible 1\&.3\&.3" "System administration commands"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
 .\" -----------------------------------------------------------------
 .\" * set default formatting
 .\" -----------------------------------------------------------------

docs/man/man1/ansible-pull.1

@@ -1,13 +1,22 @@
 '\" t
 .\"     Title: ansible
 .\"    Author: [see the "AUTHOR" section]
-.\" Generator: DocBook XSL Stylesheets v1.75.2 <http://docbook.sf.net/>
-.\"      Date: 04/02/2013
+.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
+.\"      Date: 10/07/2013
 .\"    Manual: System administration commands
-.\"    Source: Ansible 1.2
+.\"    Source: Ansible 1.3.3
 .\"  Language: English
 .\"
-.TH "ANSIBLE" "1" "04/02/2013" "Ansible 1\&.2" "System administration commands"
+.TH "ANSIBLE" "1" "10/07/2013" "Ansible 1\&.3\&.3" "System administration commands"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
 .\" -----------------------------------------------------------------
 .\" * set default formatting
 .\" -----------------------------------------------------------------
@@ -22,32 +31,55 @@
 ansible-pull \- set up a remote copy of ansible on each managed node
 .SH "SYNOPSIS"
 .sp
-ansible \-d DEST \-U URL [ \-C CHECKOUT ]
+ansible \-d DEST \-U URL [options] [ <filename\&.yml> ]
 .SH "DESCRIPTION"
 .sp
-\fBAnsible\fR is an extra\-simple tool/framework/API for doing \'remote things\' over SSH\&.
+\fBAnsible\fR is an extra\-simple tool/framework/API for doing \*(Aqremote things\*(Aq over SSH\&.
 .sp
-Use ansible\-pull to set up a remote copy of ansible on each managed node, each set to run via cron and update playbook source via git\&. This inverts the default \fBpush\fR architecture of ansible into a \fBpull\fR architecture, which has near\-limitless scaling potential\&.
+Use ansible\-pull to set up a remote copy of ansible on each managed node, each set to run via cron and update playbook source via a source repository\&. This inverts the default \fBpush\fR architecture of ansible into a \fBpull\fR architecture, which has near\-limitless scaling potential\&.
 .sp
 The setup playbook can be tuned to change the cron frequency, logging locations, and parameters to ansible\-pull\&.
 .sp
 This is useful both for extreme scale\-out as well as periodic remediation\&. Usage of the \fIfetch\fR module to retrieve logs from ansible\-pull runs would be an excellent way to gather and analyze remote logs from ansible\-pull\&.
+.SH "OPTIONAL ARGUMENT"
+.PP
+\fBfilename\&.yml\fR
+.RS 4
+The name of one the YAML format files to run as an ansible playbook\&. This can be a relative path within the checkout\&. If not provided, ansible\-pull will look for a playbook based on the host\(cqs fully\-qualified domain name and finally a playbook named
+\fBlocal\&.yml\fR\&.
+.RE
 .SH "OPTIONS"
 .PP
 \fB\-d\fR \fIDEST\fR, \fB\-\-directory=\fR\fIDEST\fR
 .RS 4
-Directory to checkout git repository into\&.
+Directory to checkout repository into\&.
 .RE
 .PP
 \fB\-U\fR \fIURL\fR, \fB\-\-url=\fR\fIURL\fR
 .RS 4
-URL of git repository to clone\&.
+URL of the playbook repository to checkout\&.
 .RE
 .PP
 \fB\-C\fR \fICHECKOUT\fR, \fB\-\-checkout=\fR\fICHECKOUT\fR
 .RS 4
-Branch/Tag/Commit to checkout\&. Defaults to
-\fIHEAD\fR\&.
+Branch/Tag/Commit to checkout\&. If not provided, uses default behavior of module used to check out playbook repository\&.
+.RE
+.PP
+\fB\-i\fR \fIPATH\fR, \fB\-\-inventory=\fR\fIPATH\fR
+.RS 4
+The
+\fIPATH\fR
+to the inventory hosts file\&. This can be a relative path within the checkout\&.
+.RE
+.PP
+\fB\-\-purge\fR
+.RS 4
+Purge the checkout after the playbook is run\&.
+.RE
+.PP
+\fB\-m\fR \fINAME\fR, \fB\-\-module\-name=\fR\fINAME\fR
+.RS 4
+Module used to checkout playbook repository\&. Defaults to git\&.
 .RE
 .SH "AUTHOR"
 .sp

docs/man/man1/ansible.1

@@ -1,13 +1,22 @@
 '\" t
 .\"     Title: ansible
 .\"    Author: [see the "AUTHOR" section]
-.\" Generator: DocBook XSL Stylesheets v1.75.2 <http://docbook.sf.net/>
-.\"      Date: 05/29/2013
+.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
+.\"      Date: 10/07/2013
 .\"    Manual: System administration commands
-.\"    Source: Ansible 1.2
+.\"    Source: Ansible 1.3.3
 .\"  Language: English
 .\"
-.TH "ANSIBLE" "1" "05/29/2013" "Ansible 1\&.2" "System administration commands"
+.TH "ANSIBLE" "1" "10/07/2013" "Ansible 1\&.3\&.3" "System administration commands"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
 .\" -----------------------------------------------------------------
 .\" * set default formatting
 .\" -----------------------------------------------------------------
@@ -25,7 +34,7 @@ ansible \- run a command somewhere else
 ansible <host\-pattern> [\-f forks] [\-m module_name] [\-a args]
 .SH "DESCRIPTION"
 .sp
-\fBAnsible\fR is an extra\-simple tool/framework/API for doing \'remote things\' over SSH\&.
+\fBAnsible\fR is an extra\-simple tool/framework/API for doing \*(Aqremote things\*(Aq over SSH\&.
 .SH "ARGUMENTS"
 .PP
 \fBhost\-pattern\fR
@@ -73,7 +82,7 @@ search path to load modules from\&. The default is
 \fI/usr/share/ansible\fR\&. This can also be set with the ANSIBLE_LIBRARY environment variable\&.
 .RE
 .PP
-\fB\-a\fR \'\fIARGUMENTS\fR\', \fB\-\-args=\fR\'\fIARGUMENTS\fR\'
+\fB\-a\fR \*(Aq\fIARGUMENTS\fR\*(Aq, \fB\-\-args=\fR\*(Aq\fIARGUMENTS\fR\*(Aq
 .RS 4
 The
 \fIARGUMENTS\fR

examples/ansible.cfg

@@ -108,9 +108,22 @@ filter_plugins     = /usr/share/ansible_plugins/filter_plugins
 # paramiko on older platforms rather than removing it
 #ssh_args = -o ControlMaster=auto -o ControlPersist=60s
 
+# The path to use for the ControlPath sockets. This defaults to
+# "%(directory)s/ansible-ssh-%%h-%%p-%%r", however on some systems with
+# very long hostnames or very long path names (caused by long user names or 
+# deeply nested home directories) this can exceed the character limit on
+# file socket names (108 characters for most platforms). In that case, you 
+# may wish to shorten the string below.
+# 
+# Example: 
+# control_path = %(directory)s/%%h-%%r
+#control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r
+
 # if True, make ansible use scp if the connection type is ssh 
 # (default is sftp)
 #scp_if_ssh = True
 
 [accelerate]
 accelerate_port = 5099
+accelerate_timeout = 30
+accelerate_connect_timeout = 5.0

lib/ansible/__init__.py

@@ -14,5 +14,5 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
-__version__ = '1.3'
+__version__ = '1.3.3'
 __author__ = 'Michael DeHaan'

lib/ansible/callbacks.py

@@ -158,6 +158,9 @@ def vv(msg, host=None):
 def vvv(msg, host=None):
     return verbose(msg, host=host, caplevel=2)
 
+def vvvv(msg, host=None):
+    return verbose(msg, host=host, caplevel=3)
+
 def verbose(msg, host=None, caplevel=2):
     if utils.VERBOSITY > caplevel:
         if host is None:

lib/ansible/constants.py

@@ -47,7 +47,7 @@ def _get_config(p, section, key, env_var, default, boolean=True):
             return value
     if p is not None:
         try:
-            return p.get(section, key)
+            return p.get(section, key, raw=True)
         except:
             return default
     return default
@@ -130,9 +130,12 @@ DEFAULT_LOG_PATH               = shell_expand_path(get_config(p, DEFAULTS, 'log_
 ANSIBLE_NOCOLOR                = get_config(p, DEFAULTS, 'nocolor', 'ANSIBLE_NOCOLOR', None, boolean=True)
 ANSIBLE_NOCOWS                 = get_config(p, DEFAULTS, 'nocows', 'ANSIBLE_NOCOWS', None, boolean=True)
 ANSIBLE_SSH_ARGS               = get_config(p, 'ssh_connection', 'ssh_args', 'ANSIBLE_SSH_ARGS', None)
+ANSIBLE_SSH_CONTROL_PATH       = get_config(p, 'ssh_connection', 'control_path', 'ANSIBLE_SSH_CONTROL_PATH', "%(directory)s/ansible-ssh-%%h-%%p-%%r")
 PARAMIKO_RECORD_HOST_KEYS      = get_config(p, 'paramiko_connection', 'record_host_keys', 'ANSIBLE_PARAMIKO_RECORD_HOST_KEYS', True, boolean=True)
-ZEROMQ_PORT                    = int(get_config(p, 'fireball_connection', 'zeromq_port', 'ANSIBLE_ZEROMQ_PORT', 5099))
-ACCELERATE_PORT                = int(get_config(p, 'accelerate', 'accelerate_port', 'ACCELERATE_PORT', 5099))
+ZEROMQ_PORT                    = get_config(p, 'fireball_connection', 'zeromq_port', 'ANSIBLE_ZEROMQ_PORT', 5099, integer=True)
+ACCELERATE_PORT                = get_config(p, 'accelerate', 'accelerate_port', 'ACCELERATE_PORT', 5099, integer=True)
+ACCELERATE_TIMEOUT             = int(get_config(p, 'accelerate', 'accelerate_timeout', 'ACCELERATE_TIMEOUT', 30))
+ACCELERATE_CONNECT_TIMEOUT     = float(get_config(p, 'accelerate', 'accelerate_connect_timeout', 'ACCELERATE_CONNECT_TIMEOUT', 1.0))
 
 DEFAULT_UNDEFINED_VAR_BEHAVIOR = get_config(p, DEFAULTS, 'error_on_undefined_vars', 'ANSIBLE_ERROR_ON_UNDEFINED_VARS', True, boolean=True)
 HOST_KEY_CHECKING              = get_config(p, DEFAULTS, 'host_key_checking',  'ANSIBLE_HOST_KEY_CHECKING',    True, boolean=True)

lib/ansible/playbook/__init__.py

@@ -449,7 +449,7 @@ class PlayBook(object):
             setup_cache=self.SETUP_CACHE, callbacks=self.runner_callbacks, sudo=play.sudo, sudo_user=play.sudo_user,
             transport=play.transport, sudo_pass=self.sudo_pass, is_playbook=True, module_vars=play.vars,
             default_vars=play.default_vars, check=self.check, diff=self.diff, 
-            accelerate=play.accelerate, accelerate_port=play.accelerate_port
+            accelerate=play.accelerate, accelerate_port=play.accelerate_port,
         ).run()
         self.stats.compute(setup_results, setup=True)
 

lib/ansible/runner/__init__.py

@@ -570,6 +570,11 @@ class Runner(object):
         actual_private_key_file = inject.get('ansible_ssh_private_key_file', self.private_key_file)
 
         if self.accelerate and actual_transport != 'local':
+            #Fix to get the inventory name of the host to accelerate plugin
+            if inject.get('ansible_ssh_host', None):
+                self.accelerate_inventory_host = host
+            else:
+                self.accelerate_inventory_host = None
             # if we're using accelerated mode, force the
             # transport to accelerate
             actual_transport = "accelerate"
@@ -623,7 +628,7 @@ class Runner(object):
                 # calls just to accomodate this one case
                 actual_port = [actual_port, self.accelerate_port]
             elif actual_port is not None:
-                actual_port = int(actual_port)
+                actual_port = int(template.template(self.basedir, actual_port, inject))
         except ValueError, e:
             result = dict(failed=True, msg="FAILED: Configured port \"%s\" is not a valid port, expected integer" % actual_port)
             return ReturnData(host=host, comm_ok=False, result=result)
@@ -771,6 +776,11 @@ class Runner(object):
         if result['rc'] != 0:
             if result['rc'] == 5:
                 output = 'Authentication failure.'
+            elif result['rc'] == 255 and self.transport == 'ssh':
+                if utils.VERBOSITY > 3:
+                    output = 'SSH encountered an unknown error. The output was:\n%s' % (result['stdout']+result['stderr'])
+                else:
+                    output = 'SSH encountered an unknown error during the connection. We recommend you re-run the command using -vvvv, which will enable SSH debugging output to help diagnose the issue'
             else:
                 output = 'Authentication or permission failure.  In some cases, you may have been able to authenticate and did not have permissions on the remote directory. Consider changing the remote temp path in ansible.cfg to a path rooted in "/tmp". Failed command was: %s, exited with result %d' % (cmd, result['rc'])
             if 'stdout' in result and result['stdout'] != '':

lib/ansible/runner/connection_plugins/accelerate.py

@@ -21,7 +21,7 @@ import base64
 import socket
 import struct
 import time
-from ansible.callbacks import vvv
+from ansible.callbacks import vvv, vvvv
 from ansible.runner.connection_plugins.ssh import Connection as SSHConnection
 from ansible import utils
 from ansible import errors
@@ -73,9 +73,13 @@ class Connection(object):
             utils.AES_KEYS = self.runner.aes_keys
 
     def _execute_accelerate_module(self):
-        args = "password=%s port=%s" % (base64.b64encode(self.key.__str__()), str(self.accport))
+        args = "password=%s port=%s debug=%d" % (base64.b64encode(self.key.__str__()), str(self.accport), int(utils.VERBOSITY))
         inject = dict(password=self.key)
-        inject = utils.combine_vars(inject, self.runner.inventory.get_variables(self.host))
+        if self.runner.accelerate_inventory_host:
+            inject = utils.combine_vars(inject, self.runner.inventory.get_variables(self.runner.accelerate_inventory_host))
+        else:
+            inject = utils.combine_vars(inject, self.runner.inventory.get_variables(self.host))
+        vvvv("attempting to start up the accelerate daemon...")
         self.ssh.connect()
         tmp_path = self.runner._make_tmp_path(self.ssh)
         return self.runner._execute_module(self.ssh, tmp_path, 'accelerate', args, inject=inject)
@@ -85,20 +89,22 @@ class Connection(object):
 
         try:
             if not self.is_connected:
-                # TODO: make the timeout and retries configurable?
                 tries = 3
                 self.conn = socket.socket()
-                self.conn.settimeout(300.0)
+                self.conn.settimeout(constants.ACCELERATE_CONNECT_TIMEOUT)
+                vvvv("attempting connection to %s via the accelerated port %d" % (self.host,self.accport))
                 while tries > 0:
                     try:
                         self.conn.connect((self.host,self.accport))
                         break
                     except:
+                        vvvv("failed, retrying...")
                         time.sleep(0.1)
                         tries -= 1
                 if tries == 0:
                     vvv("Could not connect via the accelerated connection, exceeded # of tries")
                     raise errors.AnsibleError("Failed to connect")
+                self.conn.settimeout(constants.ACCELERATE_TIMEOUT)
         except:
             if allow_ssh:
                 vvv("Falling back to ssh to startup accelerated mode")
@@ -119,18 +125,24 @@ class Connection(object):
         header_len = 8 # size of a packed unsigned long long
         data = b""
         try:
+            vvvv("%s: in recv_data(), waiting for the header" % self.host)
             while len(data) < header_len:
-                d = self.conn.recv(1024)
+                d = self.conn.recv(header_len - len(data))
                 if not d:
+                    vvvv("%s: received nothing, bailing out" % self.host)
                     return None
                 data += d
+            vvvv("%s: got the header, unpacking" % self.host)
             data_len = struct.unpack('Q',data[:header_len])[0]
             data = data[header_len:]
+            vvvv("%s: data received so far (expecting %d): %d" % (self.host,data_len,len(data)))
             while len(data) < data_len:
-                d = self.conn.recv(1024)
+                d = self.conn.recv(data_len - len(data))
                 if not d:
+                    vvvv("%s: received nothing, bailing out" % self.host)
                     return None
                 data += d
+            vvvv("%s: received all of the data, returning" % self.host)
             return data
         except socket.timeout:
             raise errors.AnsibleError("timed out while waiting to receive data")
@@ -141,7 +153,7 @@ class Connection(object):
         if executable == "":
             executable = constants.DEFAULT_EXECUTABLE
 
-        if self.runner.sudo or sudoable and sudo_user:
+        if self.runner.sudo and sudoable and sudo_user:
             cmd, prompt = utils.make_sudo_cmd(sudo_user, executable, cmd)
 
         vvv("EXEC COMMAND %s" % cmd)
@@ -157,11 +169,22 @@ class Connection(object):
         if self.send_data(data):
             raise errors.AnsibleError("Failed to send command to %s" % self.host)
         
-        response = self.recv_data()
-        if not response:
-            raise errors.AnsibleError("Failed to get a response from %s" % self.host)
-        response = utils.decrypt(self.key, response)
-        response = utils.parse_json(response)
+        while True:
+            # we loop here while waiting for the response, because a 
+            # long running command may cause us to receive keepalive packets
+            # ({"pong":"true"}) rather than the response we want. 
+            response = self.recv_data()
+            if not response:
+                raise errors.AnsibleError("Failed to get a response from %s" % self.host)
+            response = utils.decrypt(self.key, response)
+            response = utils.parse_json(response)
+            if "pong" in response:
+                # it's a keepalive, go back to waiting
+                vvvv("%s: received a keepalive packet" % self.host)
+                continue
+            else:
+                vvvv("%s: received the response" % self.host)
+                break
 
         return (response.get('rc',None), '', response.get('stdout',''), response.get('stderr',''))
 

lib/ansible/runner/connection_plugins/ssh.py

@@ -26,6 +26,7 @@ import fcntl
 import hmac
 import pwd
 import gettext
+import pty
 from hashlib import sha1
 import ansible.constants as C
 from ansible.callbacks import vvv
@@ -61,7 +62,7 @@ class Connection(object):
         else:
             self.common_args += ["-o", "ControlMaster=auto",
                                  "-o", "ControlPersist=60s",
-                                 "-o", "ControlPath=%s/ansible-ssh-%%h-%%p-%%r" % self.cp_dir]
+                                 "-o", "ControlPath=%s" % (C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=self.cp_dir))]
 
         cp_in_use = False
         cp_path_set = False
@@ -72,7 +73,7 @@ class Connection(object):
                 cp_path_set = True
 
         if cp_in_use and not cp_path_set:
-            self.common_args += ["-o", "ControlPath=%s/ansible-ssh-%%h-%%p-%%r" % self.cp_dir]
+            self.common_args += ["-o", "ControlPath=%s" % (C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=self.cp_dir))]
 
         if not C.HOST_KEY_CHECKING:
             self.common_args += ["-o", "StrictHostKeyChecking=no"]
@@ -147,7 +148,13 @@ class Connection(object):
         ''' run a command on the remote host '''
 
         ssh_cmd = self._password_cmd()
-        ssh_cmd += ["ssh", "-tt", "-q"] + self.common_args
+        ssh_cmd += ["ssh", "-tt"]
+        if utils.VERBOSITY > 3:
+            ssh_cmd += ["-vvv"]
+        else:
+            ssh_cmd += ["-q"]
+        ssh_cmd += self.common_args
+
         if self.ipv6:
             ssh_cmd += ['-6']
         ssh_cmd += [self.host]
@@ -175,11 +182,11 @@ class Connection(object):
 
         try:
             # Make sure stdin is a proper (pseudo) pty to avoid: tcgetattr errors
-            import pty
             master, slave = pty.openpty()
             p = subprocess.Popen(ssh_cmd, stdin=slave,
                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)
             stdin = os.fdopen(master, 'w', 0)
+            os.close(slave)
         except:
             p = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE)

lib/ansible/utils/__init__.py

@@ -505,7 +505,7 @@ def base_parser(constants=C, usage="", output_opts=False, runas_opts=False,
 
     parser = SortedOptParser(usage, version=version("%prog"))
     parser.add_option('-v','--verbose', default=False, action="callback",
-        callback=increment_debug, help="verbose mode (-vvv for more)")
+        callback=increment_debug, help="verbose mode (-vvv for more, -vvvv to enable connection debugging)")
 
     parser.add_option('-f','--forks', dest='forks', default=constants.DEFAULT_FORKS, type='int',
         help="specify number of parallel processes to use (default=%s)" % constants.DEFAULT_FORKS)

library/packaging/apt_repository

@@ -42,6 +42,12 @@ options:
         default: "present"
         description:
             - A source string state.
+    update_cache:
+        description:
+            - Run the equivalent of C(apt-get update) if has changed.
+        required: false
+        default: "yes"
+        choices: [ "yes", "no" ]
 author: Alexander Saltanov
 version_added: "0.7"
 requirements: [ python-apt, python-pycurl ]
@@ -69,6 +75,7 @@ import re
 import tempfile
 
 try:
+    import apt
     import apt_pkg
     import aptsources.distro
     distro = aptsources.distro.get_distro()
@@ -328,6 +335,7 @@ def main():
         argument_spec=dict(
             repo=dict(required=True),
             state=dict(choices=['present', 'absent'], default='present'),
+            update_cache = dict(aliases=['update-cache'], type='bool'),
         ),
         supports_check_mode=True,
     )
@@ -340,6 +348,7 @@ def main():
 
     repo = module.params['repo']
     state = module.params['state']
+    update_cache = module.params['update_cache']
     sourceslist = None
 
     if isinstance(distro, aptsources.distro.DebianDistribution):
@@ -365,6 +374,9 @@ def main():
     if not module.check_mode and changed:
         try:
             sourceslist.save(module)
+            if update_cache:
+                cache = apt.Cache()
+                cache.update()
         except OSError as err:
             module.fail_json(msg=unicode(err))
 

library/utilities/accelerate

@@ -35,6 +35,12 @@ options:
     required: false
     default: 5099
     aliases: []
+  timeout:
+    description:
+      - The number of seconds the socket will wait for data. If none is received when the timeout value is reached, the connection will be closed.
+    required: false
+    default: 300
+    aliases: []
   minutes:
     description:
       - The I(accelerate) listener daemon is started on nodes and will stay around for
@@ -58,24 +64,25 @@ EXAMPLES = '''
       - command: /usr/bin/anything
 '''
 
-import os
-import os.path
-import tempfile
-import sys
-import shutil
-import socket
-import struct
-import time
 import base64
 import getpass
+import os
+import os.path
+import shutil
+import signal
+import socket
+import struct
+import sys
 import syslog
-import signal
+import tempfile
 import time
-import signal
 import traceback
 
 import SocketServer
 
+from datetime import datetime
+from threading import Thread
+
 syslog.openlog('ansible-%s' % os.path.basename(__file__))
 PIDFILE = os.path.expanduser("~/.accelerate.pid")
 
@@ -85,8 +92,22 @@ PIDFILE = os.path.expanduser("~/.accelerate.pid")
 # which leaves room for the TCP/IP header
 CHUNK_SIZE=10240
 
-def log(msg):
-    syslog.syslog(syslog.LOG_NOTICE|syslog.LOG_DAEMON, msg)
+# FIXME: this all should be moved to module_common, as it's 
+#        pretty much a copy from the callbacks/util code
+DEBUG_LEVEL=0
+def log(msg, cap=0):
+    global DEBUG_LEVEL
+    if cap >= DEBUG_LEVEL:
+        syslog.syslog(syslog.LOG_NOTICE|syslog.LOG_DAEMON, msg)
+
+def vv(msg):
+    log(msg, cap=2)
+
+def vvv(msg):
+    log(msg, cap=3)
+
+def vvvv(msg):
+    log(msg, cap=4)
 
 if os.path.exists(PIDFILE):
     try:
@@ -114,7 +135,7 @@ def daemonize_self(module, password, port, minutes):
     try:
         pid = os.fork()
         if pid > 0:
-            log("exiting pid %s" % pid)
+            vvv("exiting pid %s" % pid)
             # exit first parent
             module.exit_json(msg="daemonized accelerate on port %s for %s minutes" % (port, minutes))
     except OSError, e:
@@ -134,7 +155,7 @@ def daemonize_self(module, password, port, minutes):
             pid_file = open(PIDFILE, "w")
             pid_file.write("%s" % pid)
             pid_file.close()
-            log("pidfile written")
+            vvv("pidfile written")
             sys.exit(0)
     except OSError, e:
         log("fork #2 failed: %d (%s)" % (e.errno, e.strerror))
@@ -146,12 +167,25 @@ def daemonize_self(module, password, port, minutes):
     os.dup2(dev_null.fileno(), sys.stderr.fileno())
     log("daemonizing successful")
 
+class ThreadWithReturnValue(Thread):
+    def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, Verbose=None):
+        Thread.__init__(self, group, target, name, args, kwargs, Verbose)
+        self._return = None
+
+    def run(self):
+        if self._Thread__target is not None:
+            self._return = self._Thread__target(*self._Thread__args,
+                                                **self._Thread__kwargs)
+    def join(self,timeout=None):
+        Thread.join(self, timeout=timeout)
+        return self._return
+
 class ThreadedTCPServer(SocketServer.ThreadingTCPServer):
-    def __init__(self, server_address, RequestHandlerClass, module, password):
+    def __init__(self, server_address, RequestHandlerClass, module, password, timeout):
         self.module = module
         self.key = AesKey.Read(password)
         self.allow_reuse_address = True
-        self.timeout = None
+        self.timeout = timeout
         SocketServer.ThreadingTCPServer.__init__(self, server_address, RequestHandlerClass)
 
 class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
@@ -162,50 +196,85 @@ class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
     def recv_data(self):
         header_len = 8 # size of a packed unsigned long long
         data = b""
+        vvvv("in recv_data(), waiting for the header")
         while len(data) < header_len:
-            d = self.request.recv(1024)
+            d = self.request.recv(header_len - len(data))
             if not d:
+                vvv("received nothing, bailing out")
                 return None
             data += d
+        vvvv("in recv_data(), got the header, unpacking")
         data_len = struct.unpack('Q',data[:header_len])[0]
         data = data[header_len:]
+        vvvv("data received so far (expecting %d): %d" % (data_len,len(data)))
         while len(data) < data_len:
-            d = self.request.recv(1024)
+            d = self.request.recv(data_len - len(data))
             if not d:
+                vvv("received nothing, bailing out")
                 return None
             data += d
+            vvvv("data received so far (expecting %d): %d" % (data_len,len(data)))
+        vvvv("received all of the data, returning")
         return data
 
     def handle(self):
-        while True:
-            #log("waiting for data")
-            data = self.recv_data()
-            if not data:
-                break
-            try:
-                #log("got data, decrypting")
-                data = self.server.key.Decrypt(data)
-                #log("decryption done")
-            except:
-                log("bad decrypt, skipping...")
-                data2 = json.dumps(dict(rc=1))
+        try:
+            while True:
+                vvvv("waiting for data")
+                data = self.recv_data()
+                if not data:
+                    vvvv("received nothing back from recv_data(), breaking out")
+                    break
+                try:
+                    vvvv("got data, decrypting")
+                    data = self.server.key.Decrypt(data)
+                    vvvv("decryption done")
+                except:
+                    vv("bad decrypt, skipping...")
+                    data2 = json.dumps(dict(rc=1))
+                    data2 = self.server.key.Encrypt(data2)
+                    send_data(client, data2)
+                    return
+
+                vvvv("loading json from the data")
+                data = json.loads(data)
+
+                mode = data['mode']
+                response = {}
+                last_pong = datetime.now()
+                if mode == 'command':
+                    vvvv("received a command request, running it")
+                    twrv = ThreadWithReturnValue(target=self.command, args=(data,))
+                    twrv.start()
+                    response = None
+                    while twrv.is_alive():
+                        if (datetime.now() - last_pong).seconds >= 15:
+                            last_pong = datetime.now()
+                            vvvv("command still running, sending keepalive packet")
+                            data2 = json.dumps(dict(pong=True))
+                            data2 = self.server.key.Encrypt(data2)
+                            self.send_data(data2)
+                        time.sleep(0.1)
+                    response = twrv._return
+                    vvvv("thread is done, response from join was %s" % response)
+                elif mode == 'put':
+                    vvvv("received a put request, putting it")
+                    response = self.put(data)
+                elif mode == 'fetch':
+                    vvvv("received a fetch request, getting it")
+                    response = self.fetch(data)
+
+                vvvv("response result is %s" % str(response))
+                data2 = json.dumps(response)
                 data2 = self.server.key.Encrypt(data2)
-                send_data(client, data2)
-                return
-
-            #log("loading json from the data")
-            data = json.loads(data)
-
-            mode = data['mode']
-            response = {}
-            if mode == 'command':
-                response = self.command(data)
-            elif mode == 'put':
-                response = self.put(data)
-            elif mode == 'fetch':
-                response = self.fetch(data)
-
-            data2 = json.dumps(response)
+                vvvv("sending the response back to the controller")
+                self.send_data(data2)
+                vvvv("done sending the response")
+        except:
+            tb = traceback.format_exc()
+            log("encountered an unhandled exception in the handle() function")
+            log("error was:\n%s" % tb)
+            data2 = json.dumps(dict(rc=1, failed=True, msg="unhandled error in the handle() function"))
             data2 = self.server.key.Encrypt(data2)
             self.send_data(data2)
 
@@ -217,14 +286,14 @@ class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
         if 'executable' not in data:
             return dict(failed=True, msg='internal error: executable is required')
 
-        #log("executing: %s" % data['cmd'])
+        vvvv("executing: %s" % data['cmd'])
         rc, stdout, stderr = self.server.module.run_command(data['cmd'], executable=data['executable'], close_fds=True)
         if stdout is None:
             stdout = ''
         if stderr is None:
             stderr = ''
-        #log("got stdout: %s" % stdout)
-        #log("got stderr: %s" % stderr)
+        vvvv("got stdout: %s" % stdout)
+        vvvv("got stderr: %s" % stderr)
 
         return dict(rc=rc, stdout=stdout, stderr=stderr)
 
@@ -235,7 +304,7 @@ class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
         try:
             fd = file(data['in_path'], 'rb')
             fstat = os.stat(data['in_path'])
-            log("FETCH file is %d bytes" % fstat.st_size)
+            vvv("FETCH file is %d bytes" % fstat.st_size)
             while fd.tell() < fstat.st_size:
                 data = fd.read(CHUNK_SIZE)
                 last = False
@@ -276,7 +345,7 @@ class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
         final_path = None
         final_user = None
         if 'user' in data and data.get('user') != getpass.getuser():
-            log("the target user doesn't match this user, we'll move the file into place via sudo")
+            vv("the target user doesn't match this user, we'll move the file into place via sudo")
             (fd,out_path) = tempfile.mkstemp(prefix='ansible.', dir=os.path.expanduser('~/.ansible/tmp/'))
             out_fd = os.fdopen(fd, 'w', 0)
             final_path = data['out_path']
@@ -306,22 +375,15 @@ class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler):
             log("failed to put the file: %s" % tb)
             return dict(failed=True, stdout="Could not write the file")
         finally:
-            #log("wrote %d bytes" % bytes)
+            vvvv("wrote %d bytes" % bytes)
             out_fd.close()
 
         if final_path:
-            log("moving %s to %s" % (out_path, final_path))
-            args = ['sudo','cp',out_path,final_path]
-            rc, stdout, stderr = self.server.module.run_command(args, close_fds=True)
-            if rc != 0:
-                return dict(failed=True, stdout="failed to copy the file into position with sudo")
-            args = ['sudo','chown',final_user,out_path,final_path]
-            rc, stdout, stderr = self.server.module.run_command(args, close_fds=True)
-            if rc != 0:
-                return dict(failed=True, stdout="failed to chown the file via sudo")
+            vvv("moving %s to %s" % (out_path, final_path))
+            self.server.module.atomic_move(out_path, final_path)
         return dict()
 
-def daemonize(module, password, port, minutes):
+def daemonize(module, password, port, timeout, minutes):
     try:
         daemonize_self(module, password, port, minutes)
 
@@ -331,10 +393,10 @@ def daemonize(module, password, port, minutes):
         signal.signal(signal.SIGALRM, catcher)
         signal.setitimer(signal.ITIMER_REAL, 60 * minutes)
 
-        server = ThreadedTCPServer(("0.0.0.0", port), ThreadedTCPRequestHandler, module, password)
+        server = ThreadedTCPServer(("0.0.0.0", port), ThreadedTCPRequestHandler, module, password, timeout)
         server.allow_reuse_address = True
         
-        log("serving!")
+        vv("serving!")
         server.serve_forever(poll_interval=1.0)
     except Exception, e:
         tb = traceback.format_exc()
@@ -342,23 +404,30 @@ def daemonize(module, password, port, minutes):
         sys.exit(0)
 
 def main():
+    global DEBUG_LEVEL
     module = AnsibleModule(
         argument_spec = dict(
             port=dict(required=False, default=5099),
+            timeout=dict(required=False, default=300),
             password=dict(required=True),
             minutes=dict(required=False, default=30),
-        )
+            debug=dict(required=False, default=0, type='int')
+        ),
+        supports_check_mode=True
     )
 
     password  = base64.b64decode(module.params['password'])
     port      = int(module.params['port'])
+    timeout   = int(module.params['timeout'])
     minutes   = int(module.params['minutes'])
+    debug     = int(module.params['debug'])
 
     if not HAS_KEYCZAR:
         module.fail_json(msg="keyczar is not installed")
 
-    daemonize(module, password, port, minutes)
+    DEBUG_LEVEL=debug
 
+    daemonize(module, password, port, timeout, minutes)
 
 # this is magic, see lib/ansible/module_common.py
 #<<INCLUDE_ANSIBLE_MODULE_COMMON>>

library/utilities/fireball

@@ -259,7 +259,8 @@ def main():
             port=dict(required=False, default=5099),
             password=dict(required=True),
             minutes=dict(required=False, default=30),
-        )
+        ),
+        supports_check_mode=True
     )
 
     password  = base64.b64decode(module.params['password'])

packaging/debian/changelog

@@ -1,8 +1,26 @@
-ansible (1.3) unstable; urgency=low
+ansible (1.3.3) unstable; urgency=low
 
-  * 1.3 release (PENDING)
+  * 1.3.3 release
+
+ -- James Cammarata <jcammarata@ansibleworks.com>  Mon, 07 October 2013 14:00:00 -0500
+
+ansible (1.3.2) unstable; urgency=low
+
+  * 1.3.2 release
+
+ -- James Cammarata <jcammarata@ansibleworks.com>  Thu, 19 September 2013 12:00:00 -0500
+
+ansible (1.3.1) unstable; urgency=low
+
+  * 1.3.1 release
  
- -- Michael DeHaan <michael.dehaan@gmail.com>  Fri, 05 July 2013 21:45:01 -0500
+ -- James Cammarata <jcammarata@ansibleworks.com>  Mon, 16 September 2013 16:00:00 -0500
+
+ansible (1.3.0) unstable; urgency=low
+
+  * 1.3.0 release
+ 
+ -- James Cammarata <jcammarata@ansibleworks.com>  Fri, 13 September 2013 16:00:00 -0500
 
 ansible (1.2.2) unstable; urgency=low
   

packaging/debian/control

@@ -8,6 +8,6 @@ Homepage: http://ansible.github.com/
 
 Package: ansible
 Architecture: all
-Depends: python, python-support (>= 0.90), python-jinja2, python-yaml, python-paramiko
+Depends: python, python-support (>= 0.90), python-jinja2, python-yaml, python-paramiko, sshpass
 Description: Ansible Application
  Ansible is a extra-simple tool/API for doing 'parallel remote things' over SSH executing commands, running "modules", or executing larger 'playbooks' that can serve as a configuration management or deployment system.

packaging/rpm/ansible.spec

@@ -10,7 +10,7 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-buildroot
 Name: ansible
 Release: 1%{?dist}
 Summary: SSH-based configuration management, deployment, and orchestration engine
-Version: 1.3
+Version: 1.3.3
 
 Group: Development/Libraries
 License: GPLv3
@@ -24,13 +24,16 @@ BuildRequires: python26-devel
 Requires: python26-PyYAML
 Requires: python26-paramiko
 Requires: python26-jinja2
+Requires: python26-keyczar
 %else
 BuildRequires: python2-devel
 
 Requires: PyYAML
 Requires: python-paramiko
 Requires: python-jinja2
+Requires: python-keyczar
 %endif
+Requires: sshpass
 
 %description
 
@@ -40,41 +43,6 @@ over SSH and does not require any software or daemons to be installed
 on remote nodes. Extension modules can be written in any language and
 are transferred to managed machines automatically.
 
-%package fireball
-Summary: Ansible fireball transport support
-Group: Development/Libraries
-Requires: %{name} = %{version}-%{release}
-%if 0%{?rhel} && 0%{?rhel} <= 5
-Requires: python26-keyczar
-Requires: python26-zmq
-%else
-Requires: python-keyczar
-Requires: python-zmq
-%endif
-
-%description fireball
-
-Ansible can optionally use a 0MQ based transport mechanism, which is
-considerably faster than the standard ssh mechanism when there are
-multiple actions, but requires additional supporting packages.
-
-%package node-fireball
-Summary: Ansible fireball transport - node end support
-Group: Development/Libraries
-%if 0%{?rhel} && 0%{?rhel} <= 5
-Requires: python26-keyczar
-Requires: python26-zmq
-%else
-Requires: python-keyczar
-Requires: python-zmq
-%endif
-
-%description node-fireball
-
-Ansible can optionally use a 0MQ based transport mechanism, which has
-additional requirements for nodes to use.  This package includes those
-requirements.
-
 %prep
 %setup -q
 
@@ -101,39 +69,38 @@ rm -rf $RPM_BUILD_ROOT
 %{_bindir}/ansible*
 %dir %{_datadir}/ansible
 %{_datadir}/ansible/*/*
-#%{_datadir}/ansible/*/f[a-hj-z]*
-#%{_datadir}/ansible/*/file
 %config(noreplace) %{_sysconfdir}/ansible
 %doc README.md PKG-INFO COPYING
 %doc %{_mandir}/man1/ansible*
 %doc %{_mandir}/man3/ansible.*
-#%doc %{_mandir}/man3/ansible.f[a-hj-z]*
-#%doc %{_mandir}/man3/ansible.file*
 %doc examples/playbooks
 
-%files fireball
-%{_datadir}/ansible/utilities/fireball
-%doc %{_mandir}/man3/ansible.fireball.*
-
-%files node-fireball
-%doc README.md PKG-INFO COPYING
 
 %changelog
 
-* Fri Jul 05 2013 Michael DeHaan <michael.dehaan@gmail.com> - 1.3-0
-* (release pending)
+* Thu Oct 7 2013 James Cammarata <jcammarata@ansibleworks.com> - 1.3-3
+- Release 1.3.3
+
+* Thu Sep 19 2013 James Cammarata <jcammarata@ansibleworks.com> - 1.3-2
+- Release 1.3.2
+
+* Mon Sep 16 2013 James Cammarata <jcammarata@ansibleworks.com> - 1.3-1
+- Release 1.3.1
+
+* Fri Sep 13 2013 Michael DeHaan <michael.dehaan@gmail.com> - 1.3-0
+- Release 1.3.0
 
 * Thu Jul 05 2013 Michael DeHaan <michael.dehaan@gmail.com> - 1.2-2
-* Release 1.2.2
+- Release 1.2.2
 
 * Thu Jul 04 2013 Michael DeHaan <michael.dehaan@gmail.com> - 1.2-1
-* Release 1.2.1
+- Release 1.2.1
 
 * Mon Jun 10 2013 Michael DeHaan <michael.dehaan@gmail.com> - 1.2-0
-* Release 1.2
+- Release 1.2
 
 * Tue Apr 2 2013 Michael DeHaan <michael.dehaan@gmail.com> - 1.1-0
-* Release 1.1
+- Release 1.1
 
 * Fri Feb 1 2013 Michael DeHaan <michael.dehaan@gmail.com> - 1.0-0
 - Release 1.0