Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[AIRFLOW-3112] Make SFTP hook to inherit SSH hook #3945

Merged
merged 1 commit into from
Oct 12, 2018
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 53 additions & 16 deletions airflow/contrib/hooks/sftp_hook.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,14 @@
import pysftp
import logging
import datetime
from airflow.hooks.base_hook import BaseHook
from airflow.contrib.hooks.ssh_hook import SSHHook


class SFTPHook(BaseHook):
class SFTPHook(SSHHook):
"""
This hook is inherited from SSH hook. Please refer to SSH hook for the input
arguments.

Interact with SFTP. Aims to be interchangeable with FTPHook.

Pitfalls: - In contrast with FTPHook describe_directory only returns size, type and
Expand All @@ -39,32 +42,66 @@ class SFTPHook(BaseHook):
Errors that may occur throughout but should be handled downstream.
"""

def __init__(self, ftp_conn_id='sftp_default'):
self.ftp_conn_id = ftp_conn_id
def __init__(self, ftp_conn_id='sftp_default', *args, **kwargs):
kwargs['ssh_conn_id'] = ftp_conn_id
super(SFTPHook, self).__init__(*args, **kwargs)

self.conn = None
self.private_key_pass = None

if self.ssh_conn_id is not None:
conn = self.get_connection(self.ssh_conn_id)
if conn.extra is not None:
extra_options = conn.extra_dejson
if 'private_key_pass' in extra_options:
self.private_key_pass = extra_options.get('private_key_pass', None)

# For backward compatibility
# TODO: remove in Airflow 2.1
import warnings
if 'ignore_hostkey_verification' in extra_options \
and str(extra_options["ignore_hostkey_verification"])\
.lower() == 'false':
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this line should check for true instead, because the ignore_hostkey_verification option was the inverse of the no_host_key_check option. If ignore_hostkey_verification was true, no check could be performed.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops you are right. @ckljohn or @Gwildor could one of you open a small PR (using the same JIRA) to fix this?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll have a look, thanks!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ashb my PR is #4085

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks.

warnings.warn(
'Extra option `ignore_hostkey_verification` is deprecated.'
'Please use `no_host_key_check` instead.'
'This option will be removed in Airflow 2.1',
DeprecationWarning,
stacklevel=2,
)
self.no_host_key_check = False
if 'private_key' in extra_options:
warnings.warn(
'Extra option `private_key` is deprecated.'
'Please use `key_file` instead.'
'This option will be removed in Airflow 2.1',
DeprecationWarning,
stacklevel=2,
)
self.key_file = extra_options.get('private_key')

def get_conn(self):
"""
Returns an SFTP connection object
"""
if self.conn is None:
params = self.get_connection(self.ftp_conn_id)
cnopts = pysftp.CnOpts()
if ('ignore_hostkey_verification' in params.extra_dejson and
params.extra_dejson['ignore_hostkey_verification']):
if self.no_host_key_check:
cnopts.hostkeys = None
cnopts.compression = self.compress
conn_params = {
'host': params.host,
'port': params.port,
'username': params.login,
'host': self.remote_host,
'port': self.port,
'username': self.username,
'cnopts': cnopts
}
if params.password is not None:
conn_params['password'] = params.password
if 'private_key' in params.extra_dejson:
conn_params['private_key'] = params.extra_dejson['private_key']
if 'private_key_pass' in params.extra_dejson:
conn_params['private_key_pass'] = params.extra_dejson['private_key_pass']
if self.password and self.password.strip():
conn_params['password'] = self.password
if self.key_file:
conn_params['private_key'] = self.key_file
if self.private_key_pass:
conn_params['private_key_pass'] = self.private_key_pass

self.conn = pysftp.Connection(**conn_params)
return self.conn

Expand Down