Skip to content

Commit

Permalink
Beam's Python SDK harness boot entrypoint may fail when python is not…
Browse files Browse the repository at this point in the history
… accessible, but python3 is. apache#26958 fixed
  • Loading branch information
Prabin120 committed Jun 24, 2023
1 parent e07c461 commit 7638985
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 8 deletions.
4 changes: 2 additions & 2 deletions sdks/python/container/boot.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func main() {
"--container_executable=/opt/apache/beam/boot",
}
log.Printf("Starting worker pool %v: python %v", workerPoolId, strings.Join(args, " "))
if err := execx.Execute("python", args...); err != nil {
if err := execx.Execute("python3", args...); err != nil {
log.Fatalf("Python SDK worker pool exited with error: %v", err)
}
log.Print("Python SDK worker pool exited.")
Expand Down Expand Up @@ -330,7 +330,7 @@ func setupVenv(ctx context.Context, logger *tools.Logger, baseDir, workerId stri
if err := os.MkdirAll(dir, 0750); err != nil {
return "", fmt.Errorf("failed to create Python venv directory: %s", err)
}
if err := execx.Execute("python", "-m", "venv", "--system-site-packages", dir); err != nil {
if err := execx.Execute("python3", "-m", "venv", "--system-site-packages", dir); err != nil {
return "", fmt.Errorf("python venv initialization failed: %s", err)
}

Expand Down
10 changes: 5 additions & 5 deletions sdks/python/container/piputil.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,14 +38,14 @@ func pipInstallRequirements(files []string, dir, name string) error {
// option will make sure that only things staged in the worker will be
// used without following their dependencies.
args := []string{"-m", "pip", "install", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--no-index", "--no-deps", "--find-links", dir}
if err := execx.Execute("python", args...); err != nil {
if err := execx.Execute("python3", args...); err != nil {
fmt.Println("Some packages could not be installed solely from the requirements cache. Installing packages from PyPI.")
}
// The second install round opens up the search for packages on PyPI and
// also installs dependencies. The key is that if all the packages have
// been installed in the first round then this command will be a no-op.
args = []string{"-m", "pip", "install", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--find-links", dir}
return execx.Execute("python", args...)
return execx.Execute("python3", args...)
}
}
return nil
Expand Down Expand Up @@ -78,17 +78,17 @@ func pipInstallPackage(files []string, dir, name string, force, optional bool, e
// installed if necessary. This achieves our goal outlined above.
args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", "--upgrade", "--force-reinstall", "--no-deps",
filepath.Join(dir, packageSpec)}
err := execx.Execute("python", args...)
err := execx.Execute("python3", args...)
if err != nil {
return err
}
args = []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir, packageSpec)}
return execx.Execute("python", args...)
return execx.Execute("python3", args...)
}

// Case when we do not perform a forced reinstall.
args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir, packageSpec)}
return execx.Execute("python", args...)
return execx.Execute("python3", args...)
}
}
if optional {
Expand Down
2 changes: 1 addition & 1 deletion sdks/python/expansion-service-container/boot.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func launchExpansionServiceProcess() error {
os.Setenv("PATH", strings.Join([]string{filepath.Join(dir, "bin"), os.Getenv("PATH")}, ":"))

args := []string{"-m", expansionServiceEntrypoint, "-p", strconv.Itoa(*port), "--fully_qualified_name_glob", "*"}
if err := execx.Execute("python", args...); err != nil {
if err := execx.Execute("python3", args...); err != nil {
return fmt.Errorf("Could not start the expansion service: %s", err)
}

Expand Down

0 comments on commit 7638985

Please sign in to comment.