diff --git a/client.go b/client.go
index 1d9b5df..6632148 100644
--- a/client.go
+++ b/client.go
@@ -152,8 +152,10 @@ func (c *Client) InitConn(conn net.Conn) error {
 	c.setConn(conn)
 
 	// Set initial greeting timeout.
-	c.conn.SetDeadline(time.Now().Add(c.CommandTimeout))
-	defer c.conn.SetDeadline(time.Time{})
+	if t := c.CommandTimeout; t > 0 {
+		c.conn.SetDeadline(time.Now().Add(t))
+		defer c.conn.SetDeadline(time.Time{})
+	}
 
 	_, _, err := c.Text.ReadResponse(220)
 	if err != nil {
@@ -205,8 +207,10 @@ func (c *Client) Hello(localName string) error {
 // cmd is a convenience function that sends a command and returns the response
 // textproto.Error returned by c.Text.ReadResponse is converted into SMTPError.
 func (c *Client) cmd(expectCode int, format string, args ...interface{}) (int, string, error) {
-	c.conn.SetDeadline(time.Now().Add(c.CommandTimeout))
-	defer c.conn.SetDeadline(time.Time{})
+	if t := c.CommandTimeout; t > 0 {
+		c.conn.SetDeadline(time.Now().Add(t))
+		defer c.conn.SetDeadline(time.Time{})
+	}
 
 	id, err := c.Text.Cmd(format, args...)
 	if err != nil {
@@ -444,8 +448,10 @@ type dataCloser struct {
 func (d *dataCloser) Close() error {
 	d.WriteCloser.Close()
 
-	d.c.conn.SetDeadline(time.Now().Add(d.c.SubmissionTimeout))
-	defer d.c.conn.SetDeadline(time.Time{})
+	if t := d.c.SubmissionTimeout; t > 0 {
+		d.c.conn.SetDeadline(time.Now().Add(t))
+		defer d.c.conn.SetDeadline(time.Time{})
+	}
 
 	if !d.c.lmtp {
 		code, msg, err := d.c.Text.ReadResponse(250)
diff --git a/conn.go b/conn.go
index f82538d..7dd077d 100644
--- a/conn.go
+++ b/conn.go
@@ -624,7 +624,12 @@ func (c *Conn) handleStartTLS() {
 
 	// Upgrade to TLS
 	tlsConn := tls.Server(c.conn, c.server.TLSConfig)
-
+	if t := c.server.ReadTimeout; t != 0 {
+		tlsConn.SetReadDeadline(time.Now().Add(t))
+	}
+	if t := c.server.WriteTimeout; t != 0 {
+		tlsConn.SetWriteDeadline(time.Now().Add(t))
+	}
 	if err := tlsConn.Handshake(); err != nil {
 		if err == io.EOF {
 			return
@@ -983,8 +988,8 @@ func (c *Conn) greet() {
 
 func (c *Conn) WriteResponse(code int, enhCode EnhancedCode, text ...string) {
 	// TODO: error handling
-	if c.server.WriteTimeout != 0 {
-		c.conn.SetWriteDeadline(time.Now().Add(c.server.WriteTimeout))
+	if t := c.server.WriteTimeout; t != 0 {
+		c.conn.SetWriteDeadline(time.Now().Add(t))
 	}
 
 	// All responses must include an enhanced code, if it is missing - use
@@ -1011,8 +1016,8 @@ func (c *Conn) WriteResponse(code int, enhCode EnhancedCode, text ...string) {
 
 // Reads a line of input
 func (c *Conn) ReadLine() (string, error) {
-	if c.server.ReadTimeout != 0 {
-		if err := c.conn.SetReadDeadline(time.Now().Add(c.server.ReadTimeout)); err != nil {
+	if t := c.server.ReadTimeout; t != 0 {
+		if err := c.conn.SetReadDeadline(time.Now().Add(t)); err != nil {
 			return "", err
 		}
 	}