Tempate upload enhancement (ability to upload an OVF without ovf:size…

… defined in File part) (#331)

* Supports upload of ovf which has no ovf:size specified for each dependency file
* ShowUploadProgress can exit whenever task stop, such like cancelled on GUI

CHANGELOG.md

@@ -5,6 +5,7 @@
 * Added methods `adminVdc.UpdateStorageProfile` [#340](https://github.com/vmware/go-vcloud-director/pull/340)
 * Added transparent support for bearer tokens [#341](https://github.com/vmware/go-vcloud-director/pull/341)
 * Added transparent connection using `cloudapi/1.0.0/sessions` when access through `api/sessions` is disabled
+* Able to upload an OVF without ovf:size defined in File part. Some bug fix for uploading OVA/OVF. [#331](https://github.com/vmware/go-vcloud-director/pull/331)
 
 BREAKING CHANGES:
 

govcd/api_vcd_test.go

@@ -182,10 +182,11 @@ type TestConfig struct {
 		VerboseCleanup   bool   `yaml:"verboseCleanup,omitempty"`
 	} `yaml:"logging"`
 	OVA struct {
-		OvaPath        string `yaml:"ovaPath,omitempty"`
-		OvaChunkedPath string `yaml:"ovaChunkedPath,omitempty"`
-		OvaMultiVmPath string `yaml:"ovaMultiVmPath,omitempty"`
-		OvfPath        string `yaml:"ovfPath,omitempty"`
+		OvaPath            string `yaml:"ovaPath,omitempty"`
+		OvaChunkedPath     string `yaml:"ovaChunkedPath,omitempty"`
+		OvaMultiVmPath     string `yaml:"ovaMultiVmPath,omitempty"`
+		OvaWithoutSizePath string `yaml:"ovaWithoutSizePath,omitempty"`
+		OvfPath            string `yaml:"ovfPath,omitempty"`
 	} `yaml:"ova"`
 	Media struct {
 		MediaPath       string `yaml:"mediaPath,omitempty"`

govcd/catalog.go

@@ -600,7 +600,7 @@ func checkIfFileMatchesDescription(filesAbsPaths []string, fileDescription struc
 		return fmt.Errorf("file '%s' described in ovf was not found in ova", fileDescription.HREF)
 	}
 	if fileInfo, err := os.Stat(filePath); err == nil {
-		if fileInfo.Size() != int64(fileDescription.Size) {
+		if fileDescription.Size > 0 && (fileInfo.Size() != int64(fileDescription.Size)) {
 			return fmt.Errorf("file size didn't match described in ovf: %s", filePath)
 		}
 	} else {

govcd/catalog_test.go

@@ -227,7 +227,7 @@ func (vcd *TestVCD) Test_UploadOvf_ShowUploadProgress_works(check *C) {
 	check.Assert(err, IsNil)
 	AddToCleanupList(itemName, "catalogItem", vcd.org.Org.Name+"|"+vcd.config.VCD.Catalog.Name, "Test_UploadOvf")
 
-	check.Assert(string(result), Matches, ".*Upload progress 100.00%")
+	check.Assert(string(result), Matches, "(?s).*Upload progress 100.00%.*")
 
 	catalog, err = org.GetCatalogByName(vcd.config.VCD.Catalog.Name, true)
 	check.Assert(err, IsNil)
@@ -293,6 +293,15 @@ func (vcd *TestVCD) Test_UploadOvfFile(check *C) {
 	checkUploadOvf(vcd, check, vcd.config.OVA.OvfPath, vcd.config.VCD.Catalog.Name, TestUploadOvf+"7")
 }
 
+// Tests System function UploadOvf by creating catalog and
+// checking that ova file without vmdk size specified can be uploaded.
+func (vcd *TestVCD) Test_UploadOvf_withoutVMDKSize(check *C) {
+	fmt.Printf("Running: %s\n", check.TestName())
+
+	skipWhenOvaPathMissing(vcd.config.OVA.OvaWithoutSizePath, check)
+	checkUploadOvf(vcd, check, vcd.config.OVA.OvaWithoutSizePath, vcd.config.VCD.Catalog.Name, TestUploadOvf+"8")
+}
+
 func countFolders() int {
 	files, err := ioutil.ReadDir(os.TempDir())
 	if err != nil {
@@ -435,7 +444,7 @@ func (vcd *TestVCD) Test_CatalogUploadMediaImage_ShowUploadProgress_works(check
 
 	AddToCleanupList(itemName, "mediaCatalogImage", vcd.org.Org.Name+"|"+vcd.config.VCD.Catalog.Name, "Test_CatalogUploadMediaImage_ShowUploadProgress_works")
 
-	check.Assert(string(result), Matches, ".*Upload progress 100.00%")
+	check.Assert(string(result), Matches, "(?s).*Upload progress 100.00%.*")
 	catalog, err = org.GetCatalogByName(vcd.config.VCD.Catalog.Name, false)
 	check.Assert(err, IsNil)
 	check.Assert(catalog, NotNil)

govcd/sample_govcd_test_config.yaml

@@ -174,6 +174,9 @@ ova:
   # The ova with multi VMs for tests.
   ovaMultiVmPath: ../test-resources/vapp_with_3_vms.ova
   #
+  # The ova with no VMDK size in ovf for tests.
+  ovaWithoutSizePath: ../test-resources/template_without_vmdk_size.ova
+  #
   # The ovf for uploading catalog item for tests.
   ovfPath: ../test-resources/test_vapp_template_ovf/descriptor.ovf
 media:

govcd/upload.go

@@ -67,14 +67,6 @@ func uploadFile(client *Client, filePath string, uDetails uploadDetails) (int64,
 	var count int
 	var pieceSize int64
 
-	// do not allow smaller than 1kb
-	if uDetails.uploadPieceSize > 1024 && uDetails.uploadPieceSize < uDetails.fileSizeToUpload {
-		pieceSize = uDetails.uploadPieceSize
-	} else {
-		pieceSize = defaultPieceSize
-	}
-
-	util.Logger.Printf("[TRACE] Uploading will use piece size: %#v \n", pieceSize)
 	// #nosec G304 - linter does not like 'filePath' to be a variable. However this is necessary for file uploads.
 	file, err := os.Open(filePath)
 	if err != nil {
@@ -92,6 +84,26 @@ func uploadFile(client *Client, filePath string, uDetails uploadDetails) (int64,
 
 	defer file.Close()
 
+	fileSize := fileInfo.Size()
+	// when file size in OVF does not exist, use real file size instead
+	if uDetails.fileSizeToUpload == -1 {
+		uDetails.fileSizeToUpload = fileSize
+		uDetails.allFilesSize += fileSize
+	}
+	// TODO: file size in OVF maybe wrong? how to handle that?
+	if uDetails.fileSizeToUpload != fileSize {
+		fmt.Printf("WARNING：file size %d in OVF is not align with real file size %d, upload task may hung.\n",
+			uDetails.fileSizeToUpload, fileSize)
+	}
+
+	// do not allow smaller than 1kb
+	if uDetails.uploadPieceSize > 1024 && uDetails.uploadPieceSize < uDetails.fileSizeToUpload {
+		pieceSize = uDetails.uploadPieceSize
+	} else {
+		pieceSize = defaultPieceSize
+	}
+
+	util.Logger.Printf("[TRACE] Uploading will use piece size: %#v \n", pieceSize)
 	part = make([]byte, pieceSize)
 
 	for {
@@ -122,7 +134,7 @@ func uploadFile(client *Client, filePath string, uDetails uploadDetails) (int64,
 		return 0, err
 	}
 
-	return fileInfo.Size(), nil
+	return fileSize, nil
 }
 
 // Create Request with right headers and range settings. Support multi part file upload.

govcd/uploadtask.go

@@ -39,6 +39,15 @@ func (uploadTask *UploadTask) ShowUploadProgress() error {
 
 		fmt.Printf("\rUpload progress %.2f%%", uploadTask.uploadProgress.LockedGet())
 		if uploadTask.uploadProgress.LockedGet() == 100.00 {
+			fmt.Println()
+			break
+		}
+		// Upload may be cancelled by user on GUI manually, detect task status
+		if err := uploadTask.Refresh(); err != nil {
+			return err
+		}
+		if uploadTask.Task.Task.Status != "queued" && uploadTask.Task.Task.Status != "preRunning" && uploadTask.Task.Task.Status != "running" {
+			fmt.Println()
 			break
 		}
 		time.Sleep(1 * time.Second)