Skip to content

Commit

Permalink
upload template enhancement
Browse files Browse the repository at this point in the history 
Support upload ovf which has no ovf:size specified for each dependency file
ShowUploadProgress can exit whenever task stop, such like cancelled on GUI
Return status directly for reponse which has no body, such like 416, 400

Signed-off-by: Guo, Larry (NSB - CN/Qingdao) <larry.guo@nokia-sbell.com>
  • Loading branch information
Guo, Larry (NSB - CN/Qingdao) committed Aug 13, 2020
1 parent c02e2c2 commit 9d57a8f
Show file tree
Hide file tree
Showing 9 changed files with 90 additions and 48 deletions.
56 changes: 31 additions & 25 deletions govcd/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,11 @@ func ParseErr(resp *http.Response, errType error) error {
return fmt.Errorf("[ParseErr]: error parsing error body for non-200 request: %s (%+v)", err, resp)
}

// response body maybe empty for some error, such like 416, 400
if errType.Error() == "API Error: 0: " {
errType = fmt.Errorf(resp.Status)
}

return errType
}

Expand Down Expand Up @@ -297,31 +302,32 @@ func checkRespWithErrType(resp *http.Response, err, errType error) (*http.Respon
return resp, nil
// Invalid request, parse the XML error returned and return it.
case
http.StatusBadRequest, // 400
http.StatusUnauthorized, // 401
http.StatusForbidden, // 403
http.StatusNotFound, // 404
http.StatusMethodNotAllowed, // 405
http.StatusNotAcceptable, // 406
http.StatusProxyAuthRequired, // 407
http.StatusRequestTimeout, // 408
http.StatusConflict, // 409
http.StatusGone, // 410
http.StatusLengthRequired, // 411
http.StatusPreconditionFailed, // 412
http.StatusRequestEntityTooLarge, // 413
http.StatusRequestURITooLong, // 414
http.StatusUnsupportedMediaType, // 415
http.StatusLocked, // 423
http.StatusFailedDependency, // 424
http.StatusUpgradeRequired, // 426
http.StatusPreconditionRequired, // 428
http.StatusTooManyRequests, // 429
http.StatusRequestHeaderFieldsTooLarge, // 431
http.StatusUnavailableForLegalReasons, // 451
http.StatusInternalServerError, // 500
http.StatusServiceUnavailable, // 503
http.StatusGatewayTimeout: // 504
http.StatusBadRequest, // 400
http.StatusUnauthorized, // 401
http.StatusForbidden, // 403
http.StatusNotFound, // 404
http.StatusMethodNotAllowed, // 405
http.StatusNotAcceptable, // 406
http.StatusProxyAuthRequired, // 407
http.StatusRequestTimeout, // 408
http.StatusConflict, // 409
http.StatusGone, // 410
http.StatusLengthRequired, // 411
http.StatusPreconditionFailed, // 412
http.StatusRequestEntityTooLarge, // 413
http.StatusRequestURITooLong, // 414
http.StatusUnsupportedMediaType, // 415
http.StatusRequestedRangeNotSatisfiable, // 416
http.StatusLocked, // 423
http.StatusFailedDependency, // 424
http.StatusUpgradeRequired, // 426
http.StatusPreconditionRequired, // 428
http.StatusTooManyRequests, // 429
http.StatusRequestHeaderFieldsTooLarge, // 431
http.StatusUnavailableForLegalReasons, // 451
http.StatusInternalServerError, // 500
http.StatusServiceUnavailable, // 503
http.StatusGatewayTimeout: // 504
return nil, ParseErr(resp, errType)
// Unhandled response.
default:
Expand Down
9 changes: 5 additions & 4 deletions govcd/api_vcd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,10 +171,11 @@ type TestConfig struct {
VerboseCleanup bool `yaml:"verboseCleanup,omitempty"`
} `yaml:"logging"`
OVA struct {
OvaPath string `yaml:"ovaPath,omitempty"`
OvaChunkedPath string `yaml:"ovaChunkedPath,omitempty"`
OvaMultiVmPath string `yaml:"ovaMultiVmPath,omitempty"`
OvfPath string `yaml:"ovfPath,omitempty"`
OvaPath string `yaml:"ovaPath,omitempty"`
OvaChunkedPath string `yaml:"ovaChunkedPath,omitempty"`
OvaMultiVmPath string `yaml:"ovaMultiVmPath,omitempty"`
OvaWithoutSizePath string `yaml:"ovaWithoutSizePath,omitempty"`
OvfPath string `yaml:"ovfPath,omitempty"`
} `yaml:"ova"`
Media struct {
MediaPath string `yaml:"mediaPath,omitempty"`
Expand Down
27 changes: 18 additions & 9 deletions govcd/catalog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ func doesCatalogExist(check *C, org *AdminOrg) {
func (vcd *TestVCD) Test_UploadOvf(check *C) {
fmt.Printf("Running: %s\n", check.TestName())

skipWhenOvaPathMissing(vcd, check)
skipWhenOvaPathMissing(vcd.config.OVA.OvaPath, check)
checkUploadOvf(vcd, check, vcd.config.OVA.OvaPath, vcd.config.VCD.Catalog.Name, TestUploadOvf)
}

Expand All @@ -164,7 +164,7 @@ func (vcd *TestVCD) Test_UploadOvf(check *C) {
func (vcd *TestVCD) Test_UploadOvf_chunked(check *C) {
fmt.Printf("Running: %s\n", check.TestName())

skipWhenOvaPathMissing(vcd, check)
skipWhenOvaPathMissing(vcd.config.OVA.OvaChunkedPath, check)
checkUploadOvf(vcd, check, vcd.config.OVA.OvaChunkedPath, vcd.config.VCD.Catalog.Name, TestUploadOvf+"2")
}

Expand All @@ -173,7 +173,7 @@ func (vcd *TestVCD) Test_UploadOvf_chunked(check *C) {
func (vcd *TestVCD) Test_UploadOvf_progress_works(check *C) {
fmt.Printf("Running: %s\n", check.TestName())

skipWhenOvaPathMissing(vcd, check)
skipWhenOvaPathMissing(vcd.config.OVA.OvaPath, check)
itemName := TestUploadOvf + "3"

catalog, org := findCatalog(vcd, check, vcd.config.VCD.Catalog.Name)
Expand Down Expand Up @@ -202,7 +202,7 @@ func (vcd *TestVCD) Test_UploadOvf_progress_works(check *C) {
func (vcd *TestVCD) Test_UploadOvf_ShowUploadProgress_works(check *C) {
fmt.Printf("Running: %s\n", check.TestName())

skipWhenOvaPathMissing(vcd, check)
skipWhenOvaPathMissing(vcd.config.OVA.OvaPath, check)
itemName := TestUploadOvf + "4"

catalog, org := findCatalog(vcd, check, vcd.config.VCD.Catalog.Name)
Expand Down Expand Up @@ -240,7 +240,7 @@ func (vcd *TestVCD) Test_UploadOvf_ShowUploadProgress_works(check *C) {
func (vcd *TestVCD) Test_UploadOvf_error_withSameItem(check *C) {
fmt.Printf("Running: %s\n", check.TestName())

skipWhenOvaPathMissing(vcd, check)
skipWhenOvaPathMissing(vcd.config.OVA.OvaPath, check)

itemName := TestUploadOvf + "5"

Expand All @@ -264,7 +264,7 @@ func (vcd *TestVCD) Test_UploadOvf_error_withSameItem(check *C) {
func (vcd *TestVCD) Test_UploadOvf_cleaned_extracted_files(check *C) {
fmt.Printf("Running: %s\n", check.TestName())

skipWhenOvaPathMissing(vcd, check)
skipWhenOvaPathMissing(vcd.config.OVA.OvaPath, check)

itemName := TestUploadOvf + "6"

Expand All @@ -289,10 +289,19 @@ func (vcd *TestVCD) Test_UploadOvf_cleaned_extracted_files(check *C) {
func (vcd *TestVCD) Test_UploadOvfFile(check *C) {
fmt.Printf("Running: %s\n", check.TestName())

skipWhenOvaPathMissing(vcd, check)
skipWhenOvaPathMissing(vcd.config.OVA.OvfPath, check)
checkUploadOvf(vcd, check, vcd.config.OVA.OvfPath, vcd.config.VCD.Catalog.Name, TestUploadOvf+"7")
}

// Tests System function UploadOvf by creating catalog and
// checking ova file without vmdk size specified can be uploaded.
func (vcd *TestVCD) Test_UploadOvf_withoutVMDKSize(check *C) {
fmt.Printf("Running: %s\n", check.TestName())

skipWhenOvaPathMissing(vcd.config.OVA.OvaWithoutSizePath, check)
checkUploadOvf(vcd, check, vcd.config.OVA.OvaWithoutSizePath, vcd.config.VCD.Catalog.Name, TestUploadOvf+"8")
}

func countFolders() int {
files, err := ioutil.ReadDir(os.TempDir())
if err != nil {
Expand Down Expand Up @@ -349,8 +358,8 @@ func getOrg(vcd *TestVCD, check *C) *AdminOrg {
return org
}

func skipWhenOvaPathMissing(vcd *TestVCD, check *C) {
if vcd.config.OVA.OvaPath == "" || vcd.config.OVA.OvaChunkedPath == "" {
func skipWhenOvaPathMissing(ovaPath string, check *C) {
if ovaPath == "" {
check.Skip("Skipping test because no ova path given")
}
}
Expand Down
2 changes: 1 addition & 1 deletion govcd/catalogitem_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func (vcd *TestVCD) Test_GetVAppTemplate(check *C) {
// Tests System function Delete by creating catalog item and
// deleting it after.
func (vcd *TestVCD) Test_Delete(check *C) {
skipWhenOvaPathMissing(vcd, check)
skipWhenOvaPathMissing(vcd.config.OVA.OvaPath, check)
AddToCleanupList(TestDeleteCatalogItem, "catalogItem", vcd.org.Org.Name+"|"+vcd.config.VCD.Catalog.Name, "Test_Delete")

// Fetching organization
Expand Down
2 changes: 2 additions & 0 deletions govcd/sample_govcd_test_config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@
"ova": {
"ovaPath": "../test-resources/test_vapp_template.ova",
"ovaChunkedPath": "../test-resources/template_with_custom_chunk_size.ova",
"ovaMultiVmPath": "../test-resources/vapp_with_3_vms.ova",
"ovaWithoutSizePath": "../test-resources/template_without_vmdk_size.ova",
"ovfPath": "../test-resources/test_vapp_template_ovf/descriptor.ovf"
},
"media": {
Expand Down
3 changes: 3 additions & 0 deletions govcd/sample_govcd_test_config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,9 @@ ova:
# The ova with multi VMs for tests.
ovaMultiVmPath: ../test-resources/vapp_with_3_vms.ova
#
# The ova with no VMDK size in ovf for tests.
ovaWithoutSizePath: ../test-resources/template_without_vmdk_size.ova
#
# The ovf for uploading catalog item for tests.
ovfPath: ../test-resources/test_vapp_template_ovf/descriptor.ovf
media:
Expand Down
30 changes: 21 additions & 9 deletions govcd/upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,6 @@ func uploadFile(client *Client, filePath string, uDetails uploadDetails) (int64,
var count int
var pieceSize int64

// do not allow smaller than 1kb
if uDetails.uploadPieceSize > 1024 && uDetails.uploadPieceSize < uDetails.fileSizeToUpload {
pieceSize = uDetails.uploadPieceSize
} else {
pieceSize = defaultPieceSize
}

util.Logger.Printf("[TRACE] Uploading will use piece size: %#v \n", pieceSize)
// #nosec G304 - linter does not like 'filePath' to be a variable. However this is necessary for file uploads.
file, err := os.Open(filePath)
if err != nil {
Expand All @@ -92,6 +84,26 @@ func uploadFile(client *Client, filePath string, uDetails uploadDetails) (int64,

defer file.Close()

fileSize := fileInfo.Size()
// file size in OVF maybe not exist, use real file size instead
if uDetails.fileSizeToUpload == -1 {
uDetails.fileSizeToUpload = fileSize
uDetails.allFilesSize += fileSize
}
// TODO: file size in OVF maybe wrong? how to handle that?
if uDetails.fileSizeToUpload != fileSize {
fmt.Printf("WARNING：file size %d in OVF is not align with real file size %d, upload task may hung.",
uDetails.fileSizeToUpload, fileSize)
}

// do not allow smaller than 1kb
if uDetails.uploadPieceSize > 1024 && uDetails.uploadPieceSize < uDetails.fileSizeToUpload {
pieceSize = uDetails.uploadPieceSize
} else {
pieceSize = defaultPieceSize
}

util.Logger.Printf("[TRACE] Uploading will use piece size: %#v \n", pieceSize)
part = make([]byte, pieceSize)

for {
Expand Down Expand Up @@ -122,7 +134,7 @@ func uploadFile(client *Client, filePath string, uDetails uploadDetails) (int64,
return 0, err
}

return fileInfo.Size(), nil
return fileSize, nil
}

// Create Request with right headers and range settings. Support multi part file upload.
Expand Down
9 changes: 9 additions & 0 deletions govcd/uploadtask.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,15 @@ func (uploadTask *UploadTask) ShowUploadProgress() error {

fmt.Printf("\rUpload progress %.2f%%", uploadTask.uploadProgress.LockedGet())
if uploadTask.uploadProgress.LockedGet() == 100.00 {
fmt.Println()
break
}
// Upload may be cancelled by user on GUI manually, detect task status
if err := uploadTask.Refresh(); err != nil {
return err
}
if uploadTask.Task.Task.Status != "queued" && uploadTask.Task.Task.Status != "preRunning" && uploadTask.Task.Task.Status != "running" {
fmt.Println()
break
}
time.Sleep(1 * time.Second)
Expand Down
Binary file added test-resources/template_without_vmdk_size.ova
View file
Binary file not shown.

0 comments on commit 9d57a8f

Please sign in to comment.