diff --git a/cli/command/cli.go b/cli/command/cli.go
index 11db8aaa42dd..962e71cd7276 100644
--- a/cli/command/cli.go
+++ b/cli/command/cli.go
@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/docker/cli/cli/config"
@@ -139,9 +140,12 @@ func (cli *DockerCli) loadConfigFile() {
 	cli.configFile = cliconfig.LoadDefaultConfigFile(cli.err)
 }
 
+var fetchServerInfo sync.Once
+
 // ServerInfo returns the server version details for the host this client is
 // connected to
 func (cli *DockerCli) ServerInfo() ServerInfo {
+	fetchServerInfo.Do(cli.initializeFromClient)
 	return cli.serverInfo
 }
 
@@ -276,7 +280,6 @@ func (cli *DockerCli) Initialize(opts *cliflags.ClientOptions, ops ...Initialize
 			return err
 		}
 	}
-	cli.initializeFromClient()
 	return nil
 }