[21.11] elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add filebeat module and tests

nixos/doc/manual/from_md/release-notes/rl-2111.section.xml

@@ -541,6 +541,14 @@
           <link xlink:href="options.html#opt-services.prometheus.exporters.smartctl.enable">services.prometheus.exporters.smartctl</link>.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html">filebeat</link>,
+          a lightweight shipper for forwarding and centralizing log
+          data. Available as
+          <link linkend="opt-services.filebeat.enable">services.filebeat</link>.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-21.11-incompatibilities">

nixos/doc/manual/release-notes/rl-2111.section.md

@@ -149,6 +149,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [smartctl_exporter](https://github.com/prometheus-community/smartctl_exporter), a Prometheus exporter for [S.M.A.R.T.](https://en.wikipedia.org/wiki/S.M.A.R.T.) data. Available as [services.prometheus.exporters.smartctl](options.html#opt-services.prometheus.exporters.smartctl.enable).
 
+- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable).
+
 ## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
 
 - The NixOS VM test framework, `pkgs.nixosTest`/`make-test-python.nix`, now requires detaching commands such as `succeed("foo &")` and `succeed("foo | xclip -i")` to close stdout.

nixos/modules/module-list.nix

@@ -446,6 +446,7 @@
   ./services/hardware/xow.nix
   ./services/logging/SystemdJournal2Gelf.nix
   ./services/logging/awstats.nix
+  ./services/logging/filebeat.nix
   ./services/logging/fluentd.nix
   ./services/logging/graylog.nix
   ./services/logging/heartbeat.nix

nixos/modules/services/logging/filebeat.nix

@@ -0,0 +1,253 @@
+{ config, lib, utils, pkgs, ... }:
+
+let
+  inherit (lib)
+    attrValues
+    literalExpression
+    mkEnableOption
+    mkIf
+    mkOption
+    types;
+
+  cfg = config.services.filebeat;
+
+  json = pkgs.formats.json {};
+in
+{
+  options = {
+
+    services.filebeat = {
+
+      enable = mkEnableOption "filebeat";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.filebeat;
+        defaultText = literalExpression "pkgs.filebeat";
+        example = literalExpression "pkgs.filebeat7";
+        description = ''
+          The filebeat package to use.
+        '';
+      };
+
+      inputs = mkOption {
+        description = ''
+          Inputs specify how Filebeat locates and processes input data.
+
+          This is like <literal>services.filebeat.settings.filebeat.inputs</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat input.
+
+          An input type can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.inputs._name_.type"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            type = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The input type.
+
+                Look for the value after <literal>type:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            journald.id = "everything";  # Only for filebeat7
+            log = {
+              enabled = true;
+              paths = [
+                "/var/log/*.log"
+              ];
+            };
+          };
+        '';
+      };
+
+      modules = mkOption {
+        description = ''
+          Filebeat modules provide a quick way to get started
+          processing common log formats. They contain default
+          configurations, Elasticsearch ingest pipeline definitions,
+          and Kibana dashboards to help you implement and deploy a log
+          monitoring solution.
+
+          This is like <literal>services.filebeat.settings.filebeat.modules</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat module.
+
+          A module can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.modules._name_.module"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            module = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The name of the module.
+
+                Look for the value after <literal>module:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            nginx = {
+              access = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/access.log*" ];
+              };
+              error = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/error.log*" ];
+              };
+            };
+          };
+        '';
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = json.type;
+
+          options = {
+
+            output.elasticsearch.hosts = mkOption {
+              type = with types; listOf str;
+              default = [ "127.0.0.1:9200" ];
+              example = [ "myEShost:9200" ];
+              description = ''
+                The list of Elasticsearch nodes to connect to.
+
+                The events are distributed to these nodes in round
+                robin order. If one node becomes unreachable, the
+                event is automatically sent to another node. Each
+                Elasticsearch node can be defined as a URL or
+                IP:PORT. For example:
+                <literal>http://192.15.3.2</literal>,
+                <literal>https://es.found.io:9230</literal> or
+                <literal>192.24.3.2:9300</literal>. If no port is
+                specified, <literal>9200</literal> is used.
+              '';
+            };
+
+            filebeat = {
+              inputs = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Inputs specify how Filebeat locates and processes
+                  input data. Use <xref
+                  linkend="opt-services.filebeat.inputs"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+                '';
+              };
+              modules = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Filebeat modules provide a quick way to get started
+                  processing common log formats. They contain default
+                  configurations, Elasticsearch ingest pipeline
+                  definitions, and Kibana dashboards to help you
+                  implement and deploy a log monitoring solution.
+
+                  Use <xref linkend="opt-services.filebeat.modules"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+                '';
+              };
+            };
+          };
+        };
+        default = {};
+        example = literalExpression ''
+          {
+            settings = {
+              output.elasticsearch = {
+                hosts = [ "myEShost:9200" ];
+                username = "filebeat_internal";
+                password = { _secret = "/var/keys/elasticsearch_password"; };
+              };
+              logging.level = "info";
+            };
+          };
+        '';
+
+        description = ''
+          Configuration for filebeat. See
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html"/>
+          for supported values.
+
+          Options containing secret data should be set to an attribute
+          set containing the attribute <literal>_secret</literal> - a
+          string pointing to a file containing the value the option
+          should be set to. See the example to get a better picture of
+          this: in the resulting
+          <filename>filebeat.yml</filename> file, the
+          <literal>output.elasticsearch.password</literal>
+          key will be set to the contents of the
+          <filename>/var/keys/elasticsearch_password</filename> file.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs;
+    services.filebeat.settings.filebeat.modules = attrValues cfg.modules;
+
+    systemd.services.filebeat = {
+      description = "Filebeat log shipper";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
+      serviceConfig = {
+        ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" ''
+          set -euo pipefail
+
+          umask u=rwx,g=,o=
+
+          ${utils.genJqSecretsReplacementSnippet
+              cfg.settings
+              "/var/lib/filebeat/filebeat.yml"
+           }
+        '';
+        ExecStart = ''
+          ${cfg.package}/bin/filebeat -e \
+            -c "/var/lib/filebeat/filebeat.yml" \
+            --path.data "/var/lib/filebeat"
+        '';
+        Restart = "always";
+        StateDirectory = "filebeat";
+      };
+    };
+  };
+}

nixos/modules/services/logging/journalbeat.nix

@@ -28,7 +28,6 @@ in
         type = types.package;
         default = pkgs.journalbeat;
         defaultText = literalExpression "pkgs.journalbeat";
-        example = literalExpression "pkgs.journalbeat7";
         description = ''
           The journalbeat package to use
         '';