|
| 1 | +// This file is part of Substrate. |
| 2 | + |
| 3 | +// Copyright (C) 2022 Parity Technologies (UK) Ltd. |
| 4 | +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 |
| 5 | + |
| 6 | +// This program is free software: you can redistribute it and/or modify |
| 7 | +// it under the terms of the GNU General Public License as published by |
| 8 | +// the Free Software Foundation, either version 3 of the License, or |
| 9 | +// (at your option) any later version. |
| 10 | + |
| 11 | +// This program is distributed in the hope that it will be useful, |
| 12 | +// but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 14 | +// GNU General Public License for more details. |
| 15 | + |
| 16 | +// You should have received a copy of the GNU General Public License |
| 17 | +// along with this program. If not, see <https://www.gnu.org/licenses/>. |
| 18 | + |
| 19 | +use clap::Args; |
| 20 | +use nix::{errno::Errno, sys::statvfs::statvfs}; |
| 21 | +use sc_client_db::DatabaseSource; |
| 22 | +use sp_core::traits::SpawnEssentialNamed; |
| 23 | +use std::{ |
| 24 | + path::{Path, PathBuf}, |
| 25 | + time::Duration, |
| 26 | +}; |
| 27 | + |
| 28 | +const LOG_TARGET: &str = "storage-monitor"; |
| 29 | + |
| 30 | +/// Error type used in this crate. |
| 31 | +#[derive(Debug, thiserror::Error)] |
| 32 | +pub enum Error { |
| 33 | + #[error("IO Error")] |
| 34 | + IOError(#[from] Errno), |
| 35 | + #[error("Out of storage space: available {0}MB, required {1}MB")] |
| 36 | + StorageOutOfSpace(u64, u64), |
| 37 | +} |
| 38 | + |
| 39 | +/// Parameters used to create the storage monitor. |
| 40 | +#[derive(Default, Debug, Clone, Args)] |
| 41 | +pub struct StorageMonitorParams { |
| 42 | + /// Required available space on database storage. If available space for DB storage drops below |
| 43 | + /// the given threshold, node will be gracefully terminated. If `0` is given monitoring will be |
| 44 | + /// disabled. |
| 45 | + #[arg(long = "db-storage-threshold", value_name = "MB", default_value_t = 1000)] |
| 46 | + pub threshold: u64, |
| 47 | + |
| 48 | + /// How often available space is polled. |
| 49 | + #[arg(long = "db-storage-polling-period", value_name = "SECONDS", default_value_t = 5, value_parser = clap::value_parser!(u32).range(1..))] |
| 50 | + pub polling_period: u32, |
| 51 | +} |
| 52 | + |
| 53 | +/// Storage monitor service: checks the available space for the filesystem for fiven path. |
| 54 | +pub struct StorageMonitorService { |
| 55 | + /// watched path |
| 56 | + path: PathBuf, |
| 57 | + /// number of megabytes that shall be free on the filesystem for watched path |
| 58 | + threshold: u64, |
| 59 | + /// storage space polling period (seconds) |
| 60 | + polling_period: u32, |
| 61 | +} |
| 62 | + |
| 63 | +impl StorageMonitorService { |
| 64 | + /// Creates new StorageMonitorService for given client config |
| 65 | + pub fn try_spawn( |
| 66 | + parameters: StorageMonitorParams, |
| 67 | + database: DatabaseSource, |
| 68 | + spawner: &impl SpawnEssentialNamed, |
| 69 | + ) -> Result<(), Error> { |
| 70 | + Ok(match (parameters.threshold, database.path()) { |
| 71 | + (0, _) => { |
| 72 | + log::info!( |
| 73 | + target: LOG_TARGET, |
| 74 | + "StorageMonitorService: threshold `0` given, storage monitoring disabled", |
| 75 | + ); |
| 76 | + }, |
| 77 | + (_, None) => { |
| 78 | + log::warn!( |
| 79 | + target: LOG_TARGET, |
| 80 | + "StorageMonitorService: no database path to observe", |
| 81 | + ); |
| 82 | + }, |
| 83 | + (threshold, Some(path)) => { |
| 84 | + log::debug!( |
| 85 | + target: LOG_TARGET, |
| 86 | + "Initializing StorageMonitorService for db path: {:?}", |
| 87 | + path, |
| 88 | + ); |
| 89 | + |
| 90 | + Self::check_free_space(&path, threshold)?; |
| 91 | + |
| 92 | + let storage_monitor_service = StorageMonitorService { |
| 93 | + path: path.to_path_buf(), |
| 94 | + threshold, |
| 95 | + polling_period: parameters.polling_period, |
| 96 | + }; |
| 97 | + |
| 98 | + spawner.spawn_essential( |
| 99 | + "storage-monitor", |
| 100 | + None, |
| 101 | + Box::pin(storage_monitor_service.run()), |
| 102 | + ); |
| 103 | + }, |
| 104 | + }) |
| 105 | + } |
| 106 | + |
| 107 | + /// Main monitoring loop, intended to be spawned as essential task. Quits if free space drop |
| 108 | + /// below threshold. |
| 109 | + async fn run(self) { |
| 110 | + loop { |
| 111 | + tokio::time::sleep(Duration::from_secs(self.polling_period.into())).await; |
| 112 | + if Self::check_free_space(&self.path, self.threshold).is_err() { |
| 113 | + break |
| 114 | + }; |
| 115 | + } |
| 116 | + } |
| 117 | + |
| 118 | + /// Returns free space in MB, or error if statvfs failed. |
| 119 | + fn free_space(path: &Path) -> Result<u64, Error> { |
| 120 | + statvfs(path) |
| 121 | + .map(|stats| stats.blocks_available() * stats.block_size() / 1_000_000) |
| 122 | + .map_err(Error::from) |
| 123 | + } |
| 124 | + |
| 125 | + /// Checks if the amount of free space for given `path` is above given `threshold`. |
| 126 | + /// If it dropped below, error is returned. |
| 127 | + /// System errors are silently ignored. |
| 128 | + fn check_free_space(path: &Path, threshold: u64) -> Result<(), Error> { |
| 129 | + match StorageMonitorService::free_space(path) { |
| 130 | + Ok(available_space) => { |
| 131 | + log::trace!( |
| 132 | + target: LOG_TARGET, |
| 133 | + "free: {available_space} , threshold: {threshold}.", |
| 134 | + ); |
| 135 | + |
| 136 | + if available_space < threshold { |
| 137 | + log::error!(target: LOG_TARGET, "Available space {available_space}MB for path `{}` dropped below threshold: {threshold}MB , terminating...", path.display()); |
| 138 | + Err(Error::StorageOutOfSpace(available_space, threshold)) |
| 139 | + } else { |
| 140 | + Ok(()) |
| 141 | + } |
| 142 | + }, |
| 143 | + Err(e) => { |
| 144 | + log::error!(target: LOG_TARGET, "Could not read available space: {:?}.", e); |
| 145 | + Err(e) |
| 146 | + }, |
| 147 | + } |
| 148 | + } |
| 149 | +} |
0 commit comments