modmed
diff --git a/‎README.md
+4 b/‎README.md
+4
diff --git a/‎UPDATING.md
+4 b/‎UPDATING.md
+4
diff --git a/‎airflow/bin/cli.py
+2-38 b/‎airflow/bin/cli.py
+2-38
diff --git a/‎airflow/configuration.py
+49-33 b/‎airflow/configuration.py
+49-33
diff --git a/‎airflow/contrib/auth/backends/google_auth.py
+1-2 b/‎airflow/contrib/auth/backends/google_auth.py
+1-2
diff --git a/‎airflow/contrib/example_dags/example_gcp_compute.py
+108 b/‎airflow/contrib/example_dags/example_gcp_compute.py
+108
@@ -123,6 +123,7 @@ Currently **officially** using Airflow:
 1. [Branch](https://branch.io) [[@sdebarshi](https://github.com/sdebarshi), [@dmitrig01](https://github.com/dmitrig01)]
 1. [California Data Collaborative](https://github.com/California-Data-Collaborative) powered by [ARGO Labs](http://www.argolabs.org)
 1. [Carbonite](https://www.carbonite.com) [[@ajbosco](https://github.com/ajbosco)]
+1. [CarLabs](https://www.carlabs.ai/) [[@sganz](https://github.com/sganz) & [@odannyc](https://github.com/odannyc)]
 1. [CAVA](https://www.cava.com) [[@minh5](http://github.com/minh5)]
 1. [Celect](http://www.celect.com) [[@superdosh](https://github.com/superdosh) & [@chadcelect](https://github.com/chadcelect)]
 1. [Censys](https://censys.io) [[@zakird](https://github.com/zakird), [@dadrian](https://github.com/dadrian), & [@andrewsardone](https://github.com/andrewsardone)]
@@ -144,6 +145,7 @@ Currently **officially** using Airflow:
 1. [Creditas](https://www.creditas.com.br) [[@dcassiano](https://github.com/dcassiano)]
 1. [Custom Ink](https://www.customink.com/) [[@david-dalisay](https://github.com/david-dalisay), [@dmartin11](https://github.com/dmartin11) & [@mpeteuil](https://github.com/mpeteuil)]
 1. [Dailymotion](http://www.dailymotion.com/fr) [[@germaintanguy](https://github.com/germaintanguy) & [@hc](https://github.com/hc)]
+1. [Danamica](https://www.danamica.dk) [[@testvinder](https://github.com/testvinder)]
 1. [Data Reply](https://www.datareply.co.uk/) [[@kaxil](https://github.com/kaxil)]
 1. [DataCamp](https://datacamp.com/) [[@dgrtwo](https://github.com/dgrtwo)]
 1. [DataFox](https://www.datafox.com/) [[@sudowork](https://github.com/sudowork)]
@@ -167,6 +169,7 @@ Currently **officially** using Airflow:
 1. [Glassdoor](https://github.com/Glassdoor) [[@syvineckruyk](https://github.com/syvineckruyk)]
 1. [Global Fashion Group](http://global-fashion-group.com) [[@GFG](https://github.com/GFG)]
 1. [GovTech GDS](https://gds-gov.tech) [[@chrissng](https://github.com/chrissng) & [@datagovsg](https://github.com/datagovsg)]
+1. [Grab](https://www.grab.com/sg/) [[@grab](https://github.com/grab)]
 1. [Gradeup](https://gradeup.co) [[@gradeup](https://github.com/gradeup)]
 1. [Grand Rounds](https://www.grandrounds.com/) [[@richddr](https://github.com/richddr), [@timz1290](https://github.com/timz1290), [@wenever](https://github.com/@wenever), & [@runongirlrunon](https://github.com/runongirlrunon)]
 1. [Groupalia](http://es.groupalia.com) [[@jesusfcr](https://github.com/jesusfcr)]
@@ -227,6 +230,7 @@ Currently **officially** using Airflow:
 1. [OneFineStay](https://www.onefinestay.com) [[@slangwald](https://github.com/slangwald)]
 1. [Open Knowledge International](https://okfn.org) [@vitorbaptista](https://github.com/vitorbaptista)
 1. [Overstock](https://github.com/overstock) [[@mhousley](https://github.com/mhousley) & [@mct0006](https://github.com/mct0006)]
+1. [Pagar.me](https://pagar.me/) [[@pagarme](https://github.com/pagarme)
 1. [Pandora Media](https://www.pandora.com/) [[@Acehaidrey](https://github.com/Acehaidrey) & [@wolfier](https://github.com/wolfier)]
 1. [PAYMILL](https://www.paymill.com/) [[@paymill](https://github.com/paymill) & [@matthiashuschle](https://github.com/matthiashuschle)]
 1. [PayPal](https://www.paypal.com/) [[@r39132](https://github.com/r39132) & [@jhsenjaliya](https://github.com/jhsenjaliya)]
 
@@ -52,6 +52,10 @@ To delete a user:
 airflow users --delete --username jondoe
 ```
 
+### StatsD Metrics
+
+The `scheduler_heartbeat` metric has been changed from a gauge to a counter. Each loop of the scheduler will increment the counter by 1. This provides a higher degree of visibility and allows for better integration with Prometheus using the [StatsD Exporter](https://github.com/prometheus/statsd_exporter). Scheduler upness can be determined by graphing and alerting using a rate. If the scheduler goes down, the rate will drop to 0.
+
 ### Custom auth backends interface change
 
 We have updated the version of flask-login we depend upon, and as a result any
 
@@ -19,7 +19,6 @@
 # under the License.
 
 from __future__ import print_function
-from backports.configparser import NoSectionError
 import logging
 
 import os
@@ -488,24 +487,6 @@ def _run(args, dag, ti):
 
 @cli_utils.action_logging
 def run(args, dag=None):
-    # Optional sections won't log an error if they're missing in airflow.cfg.
-    OPTIONAL_AIRFLOW_CFG_SECTIONS = [
-        'atlas',
-        'celery',
-        'celery_broker_transport_options',
-        'dask',
-        'elasticsearch',
-        'github_enterprise',
-        'hive',
-        'kerberos',
-        'kubernetes',
-        'kubernetes_node_selectors',
-        'kubernetes_secrets',
-        'ldap',
-        'lineage',
-        'mesos',
-    ]
-
     if dag:
         args.dag_id = dag.dag_id
 
@@ -519,25 +500,7 @@ def run(args, dag=None):
         if os.path.exists(args.cfg_path):
             os.remove(args.cfg_path)
 
-        # Do not log these properties since some may contain passwords.
-        # This may also set default values for database properties like
-        # core.sql_alchemy_pool_size
-        # core.sql_alchemy_pool_recycle
-        for section, config in conf_dict.items():
-            for option, value in config.items():
-                try:
-                    conf.set(section, option, value)
-                except NoSectionError:
-                    no_section_msg = (
-                        'Section {section} Option {option} '
-                        'does not exist in the config!'
-                    ).format(section=section, option=option)
-
-                    if section in OPTIONAL_AIRFLOW_CFG_SECTIONS:
-                        log.debug(no_section_msg)
-                    else:
-                        log.error(no_section_msg)
-
+        conf.conf.read_dict(conf_dict, source=args.cfg_path)
         settings.configure_vars()
 
     # IMPORTANT, have to use the NullPool, otherwise, each "run" command may leave
@@ -1077,6 +1040,7 @@ def worker(args):
         'queues': args.queues,
         'concurrency': args.concurrency,
         'hostname': args.celery_hostname,
+        'loglevel': conf.get('core', 'LOGGING_LEVEL'),
     }
 
     if args.daemon:
 
@@ -130,6 +130,10 @@ class AirflowConfigParser(ConfigParser):
         ('celery', 'result_backend'),
         # Todo: remove this in Airflow 1.11
         ('celery', 'celery_result_backend'),
+        ('atlas', 'password'),
+        ('smtp', 'smtp_password'),
+        ('ldap', 'bind_password'),
+        ('kubernetes', 'git_password'),
     }
 
     # A two-level mapping of (section -> new_name -> old_name). When reading
@@ -154,9 +158,9 @@ class AirflowConfigParser(ConfigParser):
     def __init__(self, default_config=None, *args, **kwargs):
         super(AirflowConfigParser, self).__init__(*args, **kwargs)
 
-        self.defaults = ConfigParser(*args, **kwargs)
+        self.airflow_defaults = ConfigParser(*args, **kwargs)
         if default_config is not None:
-            self.defaults.read_string(default_config)
+            self.airflow_defaults.read_string(default_config)
 
         self.is_validated = False
 
@@ -246,9 +250,9 @@ def get(self, section, key, **kwargs):
                 return option
 
         # ...then the default config
-        if self.defaults.has_option(section, key):
+        if self.airflow_defaults.has_option(section, key):
             return expand_env_var(
-                self.defaults.get(section, key, **kwargs))
+                self.airflow_defaults.get(section, key, **kwargs))
 
         else:
             log.warning(
@@ -282,6 +286,10 @@ def read(self, filenames):
         super(AirflowConfigParser, self).read(filenames)
         self._validate()
 
+    def read_dict(self, *args, **kwargs):
+        super(AirflowConfigParser, self).read_dict(*args, **kwargs)
+        self._validate()
+
     def has_option(self, section, option):
         try:
             # Using self.get() to avoid reimplementing the priority order
@@ -300,8 +308,8 @@ def remove_option(self, section, option, remove_default=True):
         if super(AirflowConfigParser, self).has_option(section, option):
             super(AirflowConfigParser, self).remove_option(section, option)
 
-        if self.defaults.has_option(section, option) and remove_default:
-            self.defaults.remove_option(section, option)
+        if self.airflow_defaults.has_option(section, option) and remove_default:
+            self.airflow_defaults.remove_option(section, option)
 
     def getsection(self, section):
         """
@@ -310,10 +318,11 @@ def getsection(self, section):
         :param section: section from the config
         :return: dict
         """
-        if section not in self._sections and section not in self.defaults._sections:
+        if (section not in self._sections and
+                section not in self.airflow_defaults._sections):
             return None
 
-        _section = copy.deepcopy(self.defaults._sections[section])
+        _section = copy.deepcopy(self.airflow_defaults._sections[section])
 
         if section in self._sections:
             _section.update(copy.deepcopy(self._sections[section]))
@@ -332,47 +341,51 @@ def getsection(self, section):
             _section[key] = val
         return _section
 
-    def as_dict(self, display_source=False, display_sensitive=False):
+    def as_dict(
+            self, display_source=False, display_sensitive=False, raw=False):
         """
         Returns the current configuration as an OrderedDict of OrderedDicts.
         :param display_source: If False, the option value is returned. If True,
             a tuple of (option_value, source) is returned. Source is either
-            'airflow.cfg' or 'default'.
+            'airflow.cfg', 'default', 'env var', or 'cmd'.
         :type display_source: bool
         :param display_sensitive: If True, the values of options set by env
             vars and bash commands will be displayed. If False, those options
             are shown as '< hidden >'
         :type display_sensitive: bool
+        :param raw: Should the values be output as interpolated values, or the
+            "raw" form that can be fed back in to ConfigParser
+        :type raw: bool
         """
-        cfg = copy.deepcopy(self.defaults._sections)
-        cfg.update(copy.deepcopy(self._sections))
-
-        # remove __name__ (affects Python 2 only)
-        for options in cfg.values():
-            options.pop('__name__', None)
-
-        # add source
-        if display_source:
-            for section in cfg:
-                for k, v in cfg[section].items():
-                    cfg[section][k] = (v, 'airflow config')
+        cfg = {}
+        configs = [
+            ('default', self.airflow_defaults),
+            ('airflow.cfg', self),
+        ]
+
+        for (source_name, config) in configs:
+            for section in config.sections():
+                sect = cfg.setdefault(section, OrderedDict())
+                for (k, val) in config.items(section=section, raw=raw):
+                    if display_source:
+                        val = (val, source_name)
+                    sect[k] = val
 
         # add env vars and overwrite because they have priority
         for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
             try:
                 _, section, key = ev.split('__')
                 opt = self._get_env_var_option(section, key)
             except ValueError:
-                opt = None
-            if opt:
-                if (
-                    not display_sensitive and
-                        ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
-                    opt = '< hidden >'
-                if display_source:
-                    opt = (opt, 'env var')
-                cfg.setdefault(section.lower(), OrderedDict()).update(
-                    {key.lower(): opt})
+                continue
+            if (not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
+                opt = '< hidden >'
+            elif raw:
+                opt = opt.replace('%', '%%')
+            if display_source:
+                opt = (opt, 'env var')
+            cfg.setdefault(section.lower(), OrderedDict()).update(
+                {key.lower(): opt})
 
         # add bash commands
         for (section, key) in self.as_command_stdout:
@@ -381,8 +394,11 @@ def as_dict(self, display_source=False, display_sensitive=False):
                 if not display_sensitive:
                     opt = '< hidden >'
                 if display_source:
-                    opt = (opt, 'bash cmd')
+                    opt = (opt, 'cmd')
+                elif raw:
+                    opt = opt.replace('%', '%%')
                 cfg.setdefault(section, OrderedDict()).update({key: opt})
+                del cfg[section][key + '_cmd']
 
         return cfg
 
 
@@ -112,8 +112,7 @@ def login(self, request):
         log.debug('Redirecting user to Google login')
         return self.google_oauth.authorize(callback=url_for(
             'google_oauth_callback',
-            _external=True,
-            _scheme='https'),
+            _external=True),
             state=request.args.get('next') or request.referrer or None)
 
     def get_google_user_profile_info(self, google_token):
 
@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG that starts, stops and sets the machine type of a Google Compute
+Engine instance.
+
+This DAG relies on the following Airflow variables
+https://airflow.apache.org/concepts.html#variables
+* PROJECT_ID - Google Cloud Platform project where the Compute Engine instance exists.
+* LOCATION - Google Cloud Platform zone where the instance exists.
+* INSTANCE - Name of the Compute Engine instance.
+* SHORT_MACHINE_TYPE_NAME - Machine type resource name to set, e.g. 'n1-standard-1'.
+    See https://cloud.google.com/compute/docs/machine-types
+"""
+
+import datetime
+
+import airflow
+from airflow import models
+from airflow.contrib.operators.gcp_compute_operator import GceInstanceStartOperator, \
+    GceInstanceStopOperator, GceSetMachineTypeOperator
+
+# [START howto_operator_gce_args]
+PROJECT_ID = models.Variable.get('PROJECT_ID', '')
+LOCATION = models.Variable.get('LOCATION', '')
+INSTANCE = models.Variable.get('INSTANCE', '')
+SHORT_MACHINE_TYPE_NAME = models.Variable.get('SHORT_MACHINE_TYPE_NAME', '')
+SET_MACHINE_TYPE_BODY = {
+    'machineType': 'zones/{}/machineTypes/{}'.format(LOCATION, SHORT_MACHINE_TYPE_NAME)
+}
+
+default_args = {
+    'start_date': airflow.utils.dates.days_ago(1)
+}
+# [END howto_operator_gce_args]
+
+with models.DAG(
+    'example_gcp_compute',
+    default_args=default_args,
+    schedule_interval=datetime.timedelta(days=1)
+) as dag:
+    # [START howto_operator_gce_start]
+    gce_instance_start = GceInstanceStartOperator(
+        project_id=PROJECT_ID,
+        zone=LOCATION,
+        resource_id=INSTANCE,
+        task_id='gcp_compute_start_task'
+    )
+    # [END howto_operator_gce_start]
+    # Duplicate start for idempotence testing
+    gce_instance_start2 = GceInstanceStartOperator(
+        project_id=PROJECT_ID,
+        zone=LOCATION,
+        resource_id=INSTANCE,
+        task_id='gcp_compute_start_task2'
+    )
+    # [START howto_operator_gce_stop]
+    gce_instance_stop = GceInstanceStopOperator(
+        project_id=PROJECT_ID,
+        zone=LOCATION,
+        resource_id=INSTANCE,
+        task_id='gcp_compute_stop_task'
+    )
+    # [END howto_operator_gce_stop]
+    # Duplicate stop for idempotence testing
+    gce_instance_stop2 = GceInstanceStopOperator(
+        project_id=PROJECT_ID,
+        zone=LOCATION,
+        resource_id=INSTANCE,
+        task_id='gcp_compute_stop_task2'
+    )
+    # [START howto_operator_gce_set_machine_type]
+    gce_set_machine_type = GceSetMachineTypeOperator(
+        project_id=PROJECT_ID,
+        zone=LOCATION,
+        resource_id=INSTANCE,
+        body=SET_MACHINE_TYPE_BODY,
+        task_id='gcp_compute_set_machine_type'
+    )
+    # [END howto_operator_gce_set_machine_type]
+    # Duplicate set machine type for idempotence testing
+    gce_set_machine_type2 = GceSetMachineTypeOperator(
+        project_id=PROJECT_ID,
+        zone=LOCATION,
+        resource_id=INSTANCE,
+        body=SET_MACHINE_TYPE_BODY,
+        task_id='gcp_compute_set_machine_type2'
+    )
+
+    gce_instance_start >> gce_instance_start2 >> gce_instance_stop >> \
+        gce_instance_stop2 >> gce_set_machine_type >> gce_set_machine_type2