diff --git a/Lib/ssl.py b/Lib/ssl.py
index 3ee3d6ac0b2..8f88d15757f 100644
--- a/Lib/ssl.py
+++ b/Lib/ssl.py
@@ -113,12 +113,7 @@ class SSLSocket(socket):
                                         keyfile, certfile,
                                         cert_reqs, ssl_version, ca_certs)
             if do_handshake_on_connect:
-                timeout = self.gettimeout()
-                try:
-                    self.settimeout(None)
-                    self.do_handshake()
-                finally:
-                    self.settimeout(timeout)
+                self.do_handshake()
         self.keyfile = keyfile
         self.certfile = certfile
         self.cert_reqs = cert_reqs
diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py
index 440c80212b6..9c79af9e2ad 100644
--- a/Lib/test/test_ssl.py
+++ b/Lib/test/test_ssl.py
@@ -457,7 +457,8 @@ else:
                     asyncore.dispatcher_with_send.__init__(self, conn)
                     self.socket = ssl.wrap_socket(conn, server_side=True,
                                                   certfile=certfile,
-                                                  do_handshake_on_connect=True)
+                                                  do_handshake_on_connect=False)
+                    self._ssl_accepting = True
 
                 def readable(self):
                     if isinstance(self.socket, ssl.SSLSocket):
@@ -465,9 +466,28 @@ else:
                             self.handle_read_event()
                     return True
 
+                def _do_ssl_handshake(self):
+                    try:
+                        self.socket.do_handshake()
+                    except ssl.SSLError, err:
+                        if err.args[0] in (ssl.SSL_ERROR_WANT_READ,
+                                           ssl.SSL_ERROR_WANT_WRITE):
+                            return
+                        elif err.args[0] == ssl.SSL_ERROR_EOF:
+                            return self.handle_close()
+                        raise
+                    except socket.error, err:
+                        if err.args[0] == errno.ECONNABORTED:
+                            return self.handle_close()
+                    else:
+                        self._ssl_accepting = False
+
                 def handle_read(self):
-                    data = self.recv(1024)
-                    self.send(data.lower())
+                    if self._ssl_accepting:
+                        self._do_ssl_handshake()
+                    else:
+                        data = self.recv(1024)
+                        self.send(data.lower())
 
                 def handle_close(self):
                     self.close()
@@ -1226,6 +1246,61 @@ else:
                 server.stop()
                 server.join()
 
+        def test_handshake_timeout(self):
+            # Issue #5103: SSL handshake must respect the socket timeout
+            server = socket.socket(socket.AF_INET)
+            host = "127.0.0.1"
+            port = test_support.bind_port(server)
+            started = threading.Event()
+            finish = False
+
+            def serve():
+                server.listen(5)
+                started.set()
+                conns = []
+                while not finish:
+                    r, w, e = select.select([server], [], [], 0.1)
+                    if server in r:
+                        # Let the socket hang around rather than having
+                        # it closed by garbage collection.
+                        conns.append(server.accept()[0])
+
+            t = threading.Thread(target=serve)
+            t.start()
+            started.wait()
+
+            try:
+                try:
+                    c = socket.socket(socket.AF_INET)
+                    c.settimeout(0.2)
+                    c.connect((host, port))
+                    # Will attempt handshake and time out
+                    try:
+                        ssl.wrap_socket(c)
+                    except ssl.SSLError, e:
+                        self.assertTrue("timed out" in str(e), str(e))
+                    else:
+                        self.fail("SSLError wasn't raised")
+                finally:
+                    c.close()
+                try:
+                    c = socket.socket(socket.AF_INET)
+                    c.settimeout(0.2)
+                    c = ssl.wrap_socket(c)
+                    # Will attempt handshake and time out
+                    try:
+                        c.connect((host, port))
+                    except ssl.SSLError, e:
+                        self.assertTrue("timed out" in str(e), str(e))
+                    else:
+                        self.fail("SSLError wasn't raised")
+                finally:
+                    c.close()
+            finally:
+                finish = True
+                t.join()
+                server.close()
+
 
 def test_main(verbose=False):
     if skip_expected:
diff --git a/Misc/NEWS b/Misc/NEWS
index 0d07c73f226..e50af64fa85 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -33,6 +33,12 @@ Core and Builtins
 Library
 -------
 
+- Issue #5103: SSL handshake would ignore the socket timeout and block
+  indefinitely if the other end didn't respond.
+
+- The do_handshake() method of SSL objects now adjusts the blocking mode of
+  the SSL structure if necessary (as other methods already do).
+
 - Issue #5238: Calling makefile() on an SSL object would prevent the
   underlying socket from being closed until all objects get truely destroyed.
 
diff --git a/Modules/_ssl.c b/Modules/_ssl.c
index 4925fe70d3a..b59ee7ce655 100644
--- a/Modules/_ssl.c
+++ b/Modules/_ssl.c
@@ -445,7 +445,12 @@ static PyObject *PySSL_SSLdo_handshake(PySSLObject *self)
 {
 	int ret;
 	int err;
-	int sockstate;
+	int sockstate, nonblocking;
+
+	/* just in case the blocking state of the socket has been changed */
+	nonblocking = (self->Socket->sock_timeout >= 0.0);
+	BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking);
+	BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking);
 
 	/* Actually negotiate SSL connection */
 	/* XXX If SSL_do_handshake() returns 0, it's also a failure. */