add link check with lychee

Cargo.toml

@@ -33,8 +33,11 @@ itertools = "0.9"
 crossterm = "0.18"
 fancy-regex = "0.4"
 regex = "1.4.2"
-signal-hook = "0.1"
+signal-hook = "0.2"
 rayon = "1.5"
+lychee = "0.4"
+tokio = { version = "0.3", features = ["full"] }
+futures = "0.3"
 
 # config parsing, must be independent of features
 

src/action/mod.rs

@@ -218,6 +218,8 @@ pub enum Action {
     Fix,
     /// Reflow doc comments so they adhere to a given maximum column width.
     Reflow,
+    /// Check contained links.
+    LinkCheck,
 }
 
 impl Action {
@@ -325,7 +327,7 @@ impl Action {
     pub fn run(self, suggestions: SuggestionSet, config: &Config) -> Result<Finish> {
         match self {
             Self::Check => self.check(suggestions, config),
-            Self::Reflow => {
+            Self::LinkCheck | Self::Reflow => {
                 let n = suggestions.len();
 
                 for (origin, suggestions) in suggestions {

src/config/mod.rs

@@ -12,6 +12,7 @@ mod search_dirs;
 pub use search_dirs::*;
 
 use crate::reflow::ReflowConfig;
+use crate::linkcheck::LinkCheckConfig;
 use crate::Detector;
 use anyhow::{anyhow, bail, Error, Result};
 use fancy_regex::Regex;
@@ -35,8 +36,13 @@ pub struct Config {
     #[serde(alias = "Languagetool")]
     pub languagetool: Option<LanguageToolConfig>,
     #[serde(alias = "ReFlow")]
+    #[serde(alias = "reFlow")]
     #[serde(alias = "Reflow")]
     pub reflow: Option<ReflowConfig>,
+    #[serde(alias = "LinkCheck")]
+    #[serde(alias = "linkCheck")]
+    #[serde(alias = "Linkcheck")]
+    pub linkcheck: Option<LinkCheckConfig>,
 }
 
 #[derive(Debug)]
@@ -400,6 +406,7 @@ impl Config {
             Detector::Hunspell => self.hunspell.is_some(),
             Detector::LanguageTool => self.languagetool.is_some(),
             Detector::Reflow => self.reflow.is_some(),
+            Detector::Lychee => true,
             #[cfg(test)]
             Detector::Dummy => true,
         }
@@ -429,6 +436,7 @@ impl Default for Config {
             // experimental and requires additional setup
             languagetool: None,
             reflow: Some(ReflowConfig::default()),
+            linkcheck: None,
         }
     }
 }

src/linkcheck/config.rs

@@ -0,0 +1,18 @@
+//! Link check configuration.
+use serde::{Deserialize, Serialize};
+
+/// Parameters for wrapping doc comments
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct LinkCheckConfig {
+    /// Ignore private ip ranges.
+    #[serde(default)]
+    pub(crate) exclude_private_ips: bool,
+}
+
+impl Default for LinkCheckConfig {
+    fn default() -> Self {
+        Self {
+            exclude_private_ips: true,
+        }
+    }
+}

src/linkcheck/mod.rs

@@ -0,0 +1,90 @@
+//! Check links of comments if they actually exist.
+
+use anyhow::{anyhow, Result};
+
+use crate::checker::Checker;
+use crate::documentation::{CheckableChunk, Documentation};
+use crate::util::{
+    byte_range_to_char_range, byte_range_to_char_range_many, load_span_from, sub_char_range,
+};
+use crate::{CommentVariant, ContentOrigin, Detector, Range, Span, LineColumn, Suggestion, SuggestionSet};
+
+
+mod config;
+pub use config::LinkCheckConfig;
+
+use tokio::prelude::*;
+use tokio::runtime::Runtime;
+
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
+
+#[derive(Debug)]
+pub struct LinkCheck;
+
+impl Checker for LinkCheck {
+    type Config = LinkCheckConfig;
+    fn check<'a, 's>(docu: &'a Documentation, config: &Self::Config) -> Result<SuggestionSet<'s>>
+    where
+        'a: 's,
+    {
+        let client = lychee::ClientBuilder::default()
+            .exclude_private_ips(config.exclude_private_ips)
+            .build().unwrap();
+
+        use std::pin::Pin;
+
+        let mut unordered: FuturesUnordered<Pin<Box<dyn futures::Future<Output=Result<SuggestionSet<'s>>>>>> = FuturesUnordered::new();
+        for (origin, chunks) in docu.iter() {
+            for chunk in chunks {
+                let client = client.clone();
+                let fut = Box::pin(async move {
+                    let ic = lychee::collector::InputContent::from_string(chunk.as_str(), lychee::extract::FileType::Markdown);
+
+                    let links = lychee::collector::collect_links(
+                        &[ic.input],
+                        None,
+                        true,
+                        5_usize,
+                    ).await.unwrap();
+
+                    let mut suggestion_set = SuggestionSet::<'s>::new();
+                    for link in links {
+                        let response = client.check(link).await;
+                        if !response.status.is_success() {
+                            let suggestion = Suggestion::<'s> {
+                                detector: Detector::Lychee,
+                                chunk,
+                                origin: origin.clone(),
+                                span: Span {
+                                    start: LineColumn {
+                                        line: 1usize,
+                                        column: 1usize,
+                                    },
+                                    end: LineColumn {
+                                        line: 1usize,
+                                        column: 1usize,
+                                    }
+                                }, // FIXME
+                                range: 0..1,
+                                replacements: vec![],
+                                description: None,
+                            };
+                            suggestion_set.add(origin.clone(), suggestion);
+                        }
+                    }
+                    Ok(suggestion_set)
+                });
+                unordered.push(fut);
+            }
+        }
+
+        let rt  = Runtime::new()?;
+
+        rt.block_on(async {
+            unordered.into_future().await;
+        });
+
+        anyhow::bail!("join all future sets")
+    }
+}

src/main.rs

@@ -12,6 +12,7 @@ mod checker;
 mod config;
 mod documentation;
 mod reflow;
+mod linkcheck;
 mod span;
 mod suggestion;
 mod traverse;
@@ -46,6 +47,7 @@ Usage:
     cargo-spellcheck [(-v...|-q)] check [--cfg=<cfg>] [--code=<code>] [--skip-readme] [--checkers=<checkers>] [[--recursive] <paths>... ]
     cargo-spellcheck [(-v...|-q)] fix [--cfg=<cfg>] [--code=<code>] [--skip-readme] [--checkers=<checkers>] [[--recursive] <paths>... ]
     cargo-spellcheck [(-v...|-q)] reflow [--cfg=<cfg>] [--code=<code>] [--skip-readme] [[--recursive] <paths>... ]
+    cargo-spellcheck [(-v...|-q)] linkcheck [--cfg=<cfg>] [--code=<code>] [--skip-readme] [[--recursive] <paths>... ]
     cargo-spellcheck [(-v...|-q)] config (--user|--stdout|--cfg=<cfg>) [--force]
     cargo-spellcheck [(-v...|-q)] [--cfg=<cfg>] [--fix] [--code=<code>] [--skip-readme] [--checkers=<checkers>] [[--recursive] <paths>... ]
     cargo-spellcheck --help
@@ -115,6 +117,7 @@ struct Args {
     cmd_fix: bool,
     cmd_check: bool,
     cmd_reflow: bool,
+    cmd_linkcheck: bool,
     cmd_config: bool,
 }
 
@@ -366,6 +369,8 @@ fn run() -> anyhow::Result<ExitCode> {
         Action::Fix
     } else if args.cmd_reflow {
         Action::Reflow
+    } else if args.cmd_linkcheck {
+        Action::LinkCheck
     } else {
         Action::Check
     };
@@ -383,6 +388,9 @@ fn run() -> anyhow::Result<ExitCode> {
         Action::Reflow => {
             reflow::Reflow::check(&combined, &config.reflow.clone().unwrap_or_default())?
         }
+        Action::LinkCheck => {
+            linkcheck::LinkCheck::check(&combined, &config.linkcheck.clone().unwrap_or_default())?
+        }
         Action::Check | Action::Fix => checker::check(&combined, &config)?,
     };
 

src/suggestion.rs

@@ -25,14 +25,16 @@ use crate::{Range, Span};
 #[repr(u8)]
 pub enum Detector {
     /// Hunspell lib based detector.
-    Hunspell = 0b0001,
+    Hunspell = 0b00001,
     /// Language tool server based detection.
-    LanguageTool = 0b0010,
+    LanguageTool = 0b00010,
     /// Reflow according to a given max column.
-    Reflow = 0b0100,
+    Reflow = 0b00100,
+    /// Link check according to a given max column.
+    Lychee = 0b01000,
     /// Detection of nothing, a test helper.
     #[cfg(test)]
-    Dummy = 0b1000,
+    Dummy = 0b10000,
 }
 
 /// Terminal size in characters.
@@ -73,6 +75,7 @@ impl fmt::Display for Detector {
             Self::LanguageTool => "LanguageTool",
             Self::Hunspell => "Hunspell",
             Self::Reflow => "Reflow",
+            Self::Lychee => "LinkCheck",
             #[cfg(test)]
             Self::Dummy => "Dummy",
         })