Update builder.sh

- Now we use podman, so checking por docker service doesn't makes sense
- Use cross for everything, otherwise the build ends up in glibc versions mismatch

.github/workflows/make-release.yml

@@ -0,0 +1,128 @@
+name: Create and upload artifacts
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-nix:
+    env:
+      IN_PIPELINE: true
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        type: [ubuntu-x64, ubuntu-x86, armv7, aarch64]
+        include:
+          - type: ubuntu-x64
+            os: ubuntu-latest
+            target: x86_64-unknown-linux-musl
+            name: fhc-linux
+            path: target/x86_64-unknown-linux-musl/release/fhc
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
+          - type: ubuntu-x86
+            os: ubuntu-latest
+            target: i686-unknown-linux-musl
+            name: fhc-linux-i386
+            path: target/i686-unknown-linux-musl/release/fhc
+            pkg_config_path: /usr/lib/i686-linux-gnu/pkgconfig
+          - type: armv7
+            os: ubuntu-latest
+            target: armv7-unknown-linux-gnueabihf
+            name: fhc-armv7
+            path: target/armv7-unknown-linux-gnueabihf/release/fhc
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
+          - type: aarch64
+            os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+            name: fhc-aarch64
+            path: target/aarch64-unknown-linux-gnu/release/fhc
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install System Dependencies
+        run: |
+          env
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: ${{ matrix.target }}
+          override: true
+      - uses: actions-rs/cargo@v1
+        env:
+          PKG_CONFIG_PATH: ${{ matrix.pkg_config_path }}
+          OPENSSL_DIR: /usr/lib/ssl
+        with:
+          use-cross: true
+          command: build
+          args: --release --target=${{ matrix.target }}
+      - name: Strip symbols from binary
+        run: |
+          strip -s ${{ matrix.path }} || arm-linux-gnueabihf-strip -s ${{ matrix.path }} || aarch64-linux-gnu-strip -s ${{ matrix.path }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.name }}
+          path: ${{ matrix.path }}
+
+  build-macos:
+    env:
+      IN_PIPELINE: true
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: x86_64-apple-darwin
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: --release --target=x86_64-apple-darwin
+      - name: Strip symbols from binary
+        run: |
+          strip -u -r target/x86_64-apple-darwin/release/fhc
+      - uses: actions/upload-artifact@v2
+        with:
+          name: fhc-osx
+          path: target/x86_64-apple-darwin/release/fhc
+      - uses: actions/upload-artifact@v2
+        with:
+          name: fhc-osx
+          path: fhc-osx
+
+  build-windows:
+    env:
+      IN_PIPELINE: true
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        type: [windows-x64, windows-x86]
+        include:
+          - type: windows-x64
+            os: windows-latest
+            target: x86_64-pc-windows-msvc
+            name: fhc-windows.exe
+            path: target\x86_64-pc-windows-msvc\release\fhc.exe
+          - type: windows-x86
+            os: windows-latest
+            target: i686-pc-windows-msvc
+            name: fhc-windows-i686.exe
+            path: target\i686-pc-windows-msvc\release\fhc.exe
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: ${{ matrix.target }}
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: --release --target=${{ matrix.target }}
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.name }}
+          path: ${{ matrix.path }}

.gitignore

@@ -1,3 +1,5 @@
 /target
 /hosts.txt
 *.txt
+armbuilder.sh
+/ghbinaries

.rustfmt.toml

@@ -0,0 +1,12 @@
+# How imports should be grouped into use statements. Imports will be merged or split to the configured level of granularity.
+imports_granularity = "One"
+# Unix or Windows line endings
+newline_style = "Unix"
+# Convert /* */ comments to // comments where possible
+normalize_comments = true
+# Convert #![doc] and #[doc] attributes to //! and /// doc comments.
+normalize_doc_attributes = true
+# Remove nested parens.
+remove_nested_parens = true
+# Reorder impl items. type and const are put first, then macros and methods.
+reorder_impl_items = true

.vscode/launch.json

@@ -0,0 +1,19 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "lldb",
+            "request": "launch",
+            "name": "Debug",
+            "program": "${workspaceRoot}/target/debug/${workspaceRootFolderName}",
+            "args": "",
+            "cwd": "${workspaceRoot}",
+            "sourceLanguages": [
+                "rust"
+            ],
+        }
+    ]
+}

.vscode/tasks.json

@@ -0,0 +1,15 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "build",
+            "type": "shell",
+            "command": "cargo build",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "problemMatcher": []
+        }
+    ]
+}

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "fhc"
-version = "0.6.1"
+version = "0.9.1"
 authors = ["Eduard Tolosa <edu4rdshl@protonmail.com>"]
 edition = "2018"
 description = "Fast HTTP Checker."
@@ -12,21 +12,21 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-tokio = { version = "1.16.1", features = ["full", "io-util"] }
+tokio = { version = "1.47.0", features = ["full", "io-util"] }
-futures = "0.3.21"
+futures = "0.3.31"
-clap = "2.33.4"
+clap = { version = "4.5.41", features = ["derive"] }
-reqwest = { version = "0.11.9", features = ["trust-dns", "rustls-tls", "native-tls"] }
+reqwest = { version = "0.12.22", features = [
-openssl = { version = "0.10.38", features = ["vendored"] }
+    "hickory-dns",
-rand = "0.8.4"
+    "rustls-tls",
-
+    "native-tls",
-# https://github.com/bluejekyll/trust-dns/pull/1632
+] }
-[patch.crates-io]
+openssl = { version = "0.10.73", features = ["vendored"] }
-trust-dns-resolver = { git = "https://github.com/Findomain/trust-dns", package = "trust-dns-resolver", branch = "custombranch" }
+rand = "0.9.2"
-
+scraper = "0.23.1"
+async-recursion = "1.1.1"
 
 [profile.release]
 lto = true
 codegen-units = 1
 panic = 'abort'
-incremental = false
+strip = true
-opt-level = "s"

builder.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# Rusolver releaser
 NAME="fhc"
 
 LINUX_TARGET="x86_64-unknown-linux-musl"
@@ -7,16 +6,21 @@ LINUX_X86_TARGET="i686-unknown-linux-musl"
 WIN_TARGET="x86_64-pc-windows-gnu"
 ARMV7_TARGET="armv7-unknown-linux-gnueabihf"
 AARCH_TARGET="aarch64-unknown-linux-gnu"
-OSX_TARGET="x86_64-apple-darwin"
+# OSX_TARGET="x86_64-apple-darwin"
 MANPAGE_DIR="./$NAME.1"
+BIN_OUTPUT_DIR="./ghbinaries"
+
+rm -rf "$BIN_OUTPUT_DIR"
+mkdir -p "$BIN_OUTPUT_DIR"
 
 # Linux build
 echo "Building Linux artifact."
-if cargo build -q --release --target="$LINUX_TARGET"; then
+if cross build -q --release --target="$LINUX_TARGET"; then
   echo "Linux artifact build: SUCCESS"
   cp "target/$LINUX_TARGET/release/$NAME" "target/$LINUX_TARGET/release/$NAME-linux"
   strip "target/$LINUX_TARGET/release/$NAME-linux"
-  sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"target/$LINUX_TARGET/release/$NAME-linux.sha512"
+  sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"$BIN_OUTPUT_DIR/$NAME-linux.sha512"
+  zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-x64.zip" "target/$LINUX_TARGET/release/$NAME-linux"
 else
   echo "Linux artifact build: FAILED"
 fi
@@ -27,7 +31,8 @@ if cross build -q --release --target="$LINUX_X86_TARGET"; then
   echo "Linux x86 artifact build: SUCCESS"
   cp "target/$LINUX_X86_TARGET/release/$NAME" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
   strip "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
-  sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"target/$LINUX_X86_TARGET/release/$NAME-linux-i386.sha512"
+  sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"$BIN_OUTPUT_DIR/$NAME-linux-i386.sha512"
+  zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-i386.zip" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
 else
   echo "Linux x86 artifact build: FAILED"
 fi
@@ -38,7 +43,8 @@ if cross build -q --release --target="$WIN_TARGET"; then
   echo "Windows artifact build: SUCCESS"
   cp "target/$WIN_TARGET/release/$NAME.exe" "target/$WIN_TARGET/release/$NAME-windows.exe"
   strip "target/$WIN_TARGET/release/$NAME-windows.exe"
-  sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"target/$WIN_TARGET/release/$NAME-windows.exe.sha512"
+  sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"$BIN_OUTPUT_DIR/$NAME-windows.exe.sha512"
+  zip -q -j "$BIN_OUTPUT_DIR/$NAME-windows.zip" "target/$WIN_TARGET/release/$NAME-windows.exe"
 else
   echo "Windows artifact build: FAILED"
 fi
@@ -49,7 +55,8 @@ if cross build -q --release --target="$ARMV7_TARGET"; then
   echo "ARMv7 artifact build: SUCCESS"
   cp "target/$ARMV7_TARGET/release/$NAME" "target/$ARMV7_TARGET/release/$NAME-armv7"
   strip "target/$ARMV7_TARGET/release/$NAME-armv7"
-  sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"target/$ARMV7_TARGET/release/$NAME-armv7.sha512"
+  sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"$BIN_OUTPUT_DIR/$NAME-armv7.sha512"
+  zip -q -j "$BIN_OUTPUT_DIR/$NAME-armv7.zip" "target/$ARMV7_TARGET/release/$NAME-armv7"
 else
   echo "ARMv7 artifact build: FAILED"
 fi
@@ -60,21 +67,23 @@ if cross build -q --release --target="$AARCH_TARGET"; then
   echo "Aarch64 artifact build: SUCCESS"
   cp "target/$AARCH_TARGET/release/$NAME" "target/$AARCH_TARGET/release/$NAME-aarch64"
   strip "target/$AARCH_TARGET/release/$NAME-aarch64"
-  sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"target/$AARCH_TARGET/release/$NAME-aarch64.sha512"
+  sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"$BIN_OUTPUT_DIR/$NAME-aarch64.sha512"
+  zip -q -j "$BIN_OUTPUT_DIR/$NAME-aarch64.zip" "target/$AARCH_TARGET/release/$NAME-aarch64"
 else
   echo "Aarch64 artifact build: FAILED"
 fi
 
-# Mac OS build
+# # Mac OS build
-echo "Building OSX artifact."
+# echo "Building OSX artifact."
-if CC=o64-clang CXX=o64-clang++ LIBZ_SYS_STATIC=1 cargo build -q --release --target="$OSX_TARGET"; then
+# if CC=o64-clang CXX=o64-clang++ LIBZ_SYS_STATIC=1 cargo build -q --release --target="$OSX_TARGET"; then
-  echo "OSX artifact build: SUCCESS"
+#   echo "OSX artifact build: SUCCESS"
-  cp "target/$OSX_TARGET/release/$NAME" "target/$OSX_TARGET/release/$NAME-osx"
+#   cp "target/$OSX_TARGET/release/$NAME" "target/$OSX_TARGET/release/$NAME-osx"
-  strip "target/$OSX_TARGET/release/$NAME-osx"
+#   strip "target/$OSX_TARGET/release/$NAME-osx"
-  sha512sum "target/$OSX_TARGET/release/$NAME-osx" >"target/$OSX_TARGET/release/$NAME-osx.sha512"
+#   sha512sum "target/$OSX_TARGET/release/$NAME-osx" >"target/$OSX_TARGET/release/$NAME-osx.sha512"
-else
+#   zip -q "$BIN_OUTPUT_DIR/$NAME-osx.zip" "target/$OSX_TARGET/release/$NAME-osx"
-  echo "OSX artifact build: FAILED"
+# else
-fi
+#   echo "OSX artifact build: FAILED"
+# fi
 
 echo "Creating manpage..."
 if command -v help2man >/dev/null; then
@@ -86,19 +95,3 @@ if command -v help2man >/dev/null; then
 else
   echo "Please install the help2man package."
 fi
-
-# if command -v git >/dev/null; then
-#   git add .
-#   git commit -m "Bump version."
-#   git push
-# fi
-
-#echo "Uploading crate to crates.io..."
-#if cargo publish --no-verify > /dev/null; then
-#  echo "Crate uploaded."
-#else
-#  echo "An error has occurred while uploading the crate to crates.io."
-#  exit
-#fi
-
-echo "All builds have passed!"

fhc.1

@@ -1,61 +1,58 @@
-.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.48.5.
+.\" DO NOT MODIFY THIS FILE!  It was generated by help2man 1.49.3.
-.TH FHC "1" "February 2022" "FHC 0.6.0" "User Commands"
+.TH FHC "1" "August 2025" "fhc 0.9.1" "User Commands"
 .SH NAME
-FHC \- manual page for FHC 0.6.0
+fhc \- manual page for fhc 0.9.1
+.SH SYNOPSIS
+.B fhc
+[\fI\,OPTIONS\/\fR]
 .SH DESCRIPTION
-FHC 0.6.0
-Eduard Tolosa <edu4rdshl@protonmail.com>
 Fast HTTP Checker.
-.SS "USAGE:"
+.SH OPTIONS
-.IP
-fhc [FLAGS] [OPTIONS]
-.SS "FLAGS:"
 .TP
-\fB\-1\fR, \fB\-\-1xx\fR
+\fB\-t\fR, \fB\-\-threads\fR <THREADS>
-Show URLs with 100\-199 response codes only.
+Number of threads to use [default: 50]
 .TP
-\fB\-2\fR, \fB\-\-2xx\fR
+\fB\-\-timeout\fR <TIMEOUT>
-Show URLs with 200\-299 response codes only.
+Timeout in seconds [default: 3]
 .TP
-\fB\-3\fR, \fB\-\-3xx\fR
+\fB\-s\fR, \fB\-\-show\-full\-data\fR
-Show URLs with 300\-399 response codes only.
+Show HTTP status codes, final URL and domain
 .TP
-\fB\-4\fR, \fB\-\-4xx\fR
+\fB\-d\fR, \fB\-\-domain\fR <DOMAIN>
-Show URLs with 400\-499 response codes only.
+Domain to check \- can be omitted if using stdin
 .TP
-\fB\-5\fR, \fB\-\-5xx\fR
+\fB\-r\fR, \fB\-\-retries\fR <RETRIES>
-Show URLs with 500\-599 response codes only.
+Number of retries [default: 1]
+.TP
+\fB\-L\fR, \fB\-\-max\-redirects\fR <MAX_REDIRECTS>
+Maximum number of redirects, disabled by default [default: 0]
+.TP
+\fB\-b\fR, \fB\-\-bruteforce\fR
+Enable bruteforce mode
+.TP
+\fB\-f\fR, \fB\-\-filter\-codes\fR <FILTER_CODES>
+Filter status codes. A comma separated list can be used
+.TP
+\fB\-e\fR, \fB\-\-exclude\-codes\fR <EXCLUDE_CODES>
+Exclude status codes. A comma separated list can be used
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Quiet mode. This will suppress all fancy output except for the final results
 .TP
 \fB\-h\fR, \fB\-\-help\fR
-Prints help information
+Print help
-.TP
-\fB\-s\fR, \fB\-\-show\-codes\fR
-Show status codes for discovered hosts.
 .TP
 \fB\-V\fR, \fB\-\-version\fR
-Prints version information
+Print version
-.SS "OPTIONS:"
-.TP
-\fB\-d\fR, \fB\-\-domain\fR <domain>
-Target domain. When it's specified, a wordlist can be used from stdin for bruteforcing.
-.TP
-\fB\-r\fR, \fB\-\-retries\fR <retries>
-Max number of http probes per target.
-.TP
-\fB\-t\fR, \fB\-\-threads\fR <threads>
-Number of threads. Default: 50
-.TP
-\fB\-\-timeout\fR <timeout>
-Timeout in seconds. Default: 3
 .SH "SEE ALSO"
 The full documentation for
-.B FHC
+.B fhc
 is maintained as a Texinfo manual.  If the
 .B info
 and
-.B FHC
+.B fhc
 programs are properly installed at your site, the command
 .IP
-.B info FHC
+.B info fhc
 .PP
 should give you access to the complete manual.

src/args.rs

@@ -0,0 +1,36 @@
+use clap::Parser;
+
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+pub struct Cli {
+    #[clap(short, long, default_value = "50")]
+    /// Number of threads to use
+    pub threads: usize,
+    #[clap(long, default_value = "3")]
+    /// Timeout in seconds
+    pub timeout: u64,
+    #[clap(short, long)]
+    /// Show HTTP status codes, final URL and domain
+    pub show_full_data: bool,
+    #[clap(short, long)]
+    /// Domain to check - can be omitted if using stdin
+    pub domain: Option<String>,
+    #[clap(short, long, default_value = "1")]
+    /// Number of retries
+    pub retries: usize,
+    #[clap(short = 'L', long, default_value = "0")]
+    /// Maximum number of redirects, disabled by default.
+    pub max_redirects: usize,
+    #[clap(short, long)]
+    /// Enable bruteforce mode
+    pub bruteforce: bool,
+    #[clap(short, long)]
+    /// Filter status codes. A comma separated list can be used
+    pub filter_codes: Option<String>,
+    #[clap(short, long)]
+    /// Exclude status codes. A comma separated list can be used
+    pub exclude_codes: Option<String>,
+    #[clap(short, long)]
+    /// Quiet mode. This will suppress all fancy output except for the final results
+    pub quiet: bool,
+}

src/httplib.rs

@@ -1,92 +1,97 @@
-use std::collections::{HashMap, HashSet};
-
-use reqwest::Client;
-
 use {
-    crate::{structs::HttpData, utils},
+    crate::{
+        structs::{HTTPFilters, HttpData, LibOptions},
+        utils,
+    },
     futures::stream::StreamExt,
-    reqwest::{self, header::USER_AGENT},
+    rand::{distr::Alphanumeric, rng, Rng},
+    reqwest::{
+        header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT},
+        redirect::Policy,
+        Client, Response,
+    },
+    scraper::{Html, Selector},
+    std::collections::{HashMap, HashSet},
 };
 
-#[allow(clippy::too_many_arguments)]
+#[must_use]
-pub async fn return_http_data(
+#[allow(clippy::similar_names)]
-    hosts: HashSet<String>,
+pub async fn return_http_data(options: &LibOptions, from_cli: bool) -> HashMap<String, HttpData> {
-    client: Client,
+    let threads = options.hosts.len().min(options.threads);
-    user_agents_list: Vec<String>,
-    retries: usize,
-    mut threads: usize,
-    conditional_response_code: u16,
-    show_status_codes: bool,
-    quiet_flag: bool,
-) -> HashMap<String, HttpData> {
-    if hosts.len() < threads {
-        threads = hosts.len();
-    }
 
-    futures::stream::iter(hosts.into_iter().map(|host| {
+    let filter_codes = options.filter_codes.as_deref().unwrap_or_default();
-        // Use a random user agent
+    let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default();
-        let user_agent = utils::return_random_string(&user_agents_list);
-        // HTTP/HTTP URLs
-        let https_url = format!("https://{}", host);
-        let http_url = format!("http://{}", host);
-        // Create futures
-        let https_send_fut = client.get(&https_url).header(USER_AGENT, &user_agent);
-        let http_send_fut = client.get(&http_url).header(USER_AGENT, &user_agent);
 
-        let mut http_data = HttpData::default();
+    futures::stream::iter(options.hosts.iter().map(|host| {
+        let user_agent = utils::return_random_user_agent(&options.user_agents);
 
         async move {
-            if retries != 1 {
+            let mut http_data = HttpData::new(host.clone());
-                let mut counter = 0;
+            let mut response = None;
-                while counter < retries {
+
-                    if let Ok(resp) = https_send_fut
+            for _ in 0..options.retries {
-                        .try_clone()
+                let https_req = options
-                        .expect("Failed to clone https future")
+                    .client
-                        .send()
+                    .get(format!("https://{host}"))
-                        .await
+                    .header(USER_AGENT, &user_agent)
-                    {
+                    .send();
-                        http_data.host_url = https_url.clone();
+
-                        http_data.status_code = resp.status().as_u16();
+                let http_req = options
-                        http_data.http_status = "ACTIVE".to_string();
+                    .client
-                        drop(resp)
+                    .get(format!("http://{host}"))
-                    } else if let Ok(resp) = http_send_fut
+                    .header(USER_AGENT, &user_agent)
-                        .try_clone()
+                    .send();
-                        .expect("Failed to clone http future")
+
-                        .send()
+                response = tokio::select! {
-                        .await
+                    https_result = https_req => https_result.ok(),
-                    {
+                    http_result = http_req => http_result.ok(),
-                        http_data.host_url = http_url.clone();
+                };
-                        http_data.status_code = resp.status().as_u16();
+
-                        http_data.http_status = "ACTIVE".to_string();
+                if response.is_some() {
-                        drop(resp)
+                    break;
-                    }
-                    counter += 1
                 }
-            } else if let Ok(resp) = https_send_fut.send().await {
+            }
-                http_data.host_url = https_url;
+
+            if let Some(resp) = response {
+                // Those are always set
+                http_data.protocol = resp.url().scheme().to_string();
                 http_data.status_code = resp.status().as_u16();
-                http_data.http_status = "ACTIVE".to_string();
+                http_data.final_url = resp.url().to_string();
-                drop(resp)
+
-            } else if let Ok(resp) = http_send_fut.send().await {
+                if !from_cli {
-                http_data.host_url = http_url;
+                    assign_response_data(&mut http_data, resp, options.return_filters).await;
-                http_data.status_code = resp.status().as_u16();
+                }
-                http_data.http_status = "ACTIVE".to_string();
-                drop(resp)
             } else {
                 http_data.http_status = "INACTIVE".to_string();
             }
-            if !quiet_flag && (!http_data.host_url.is_empty() && conditional_response_code == 0)
+
-                || ((!http_data.host_url.is_empty() && conditional_response_code != 0)
+            if !options.quiet_flag
-                    && (http_data.status_code >= conditional_response_code
+                && !http_data.final_url.is_empty()
-                        && http_data.status_code <= conditional_response_code + 99))
+                && (filter_codes.is_empty()
+                    || filter_codes.contains(&http_data.status_code.to_string()))
+                && (exclude_codes.is_empty()
+                    || !exclude_codes.contains(&http_data.status_code.to_string()))
             {
-                if show_status_codes {
+                // Use faster I/O for high-throughput scenarios
-                    println!("{},{}", http_data.host_url, http_data.status_code)
+                use std::io::{self, Write};
+                let stdout = io::stdout();
+                let mut handle = stdout.lock();
+
+                if options.show_full_data {
+                    let _ = writeln!(
+                        handle,
+                        "{},[{}],[{}]",
+                        http_data.checked_host, http_data.final_url, http_data.status_code
+                    );
                 } else {
-                    println!("{}", http_data.host_url)
+                    let _ = writeln!(
+                        handle,
+                        "{}://{}",
+                        http_data.protocol, http_data.checked_host
+                    );
                 }
             }
-            (host, http_data)
+            (host.clone(), http_data)
         }
     }))
     .buffer_unordered(threads)
@@ -94,12 +99,141 @@ pub async fn return_http_data(
     .await
 }
 
-pub fn return_http_client(timeout: u64) -> Client {
+#[must_use]
+pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client {
+    let policy = if max_redirects == 0 {
+        Policy::none()
+    } else {
+        Policy::limited(max_redirects)
+    };
+
     reqwest::Client::builder()
         .timeout(std::time::Duration::from_secs(timeout))
+        .redirect(policy)
         .danger_accept_invalid_certs(true)
-        .trust_dns(true)
         .use_native_tls()
+        .pool_max_idle_per_host(50)
+        .pool_idle_timeout(std::time::Duration::from_secs(30))
+        .tcp_keepalive(std::time::Duration::from_secs(60))
         .build()
         .expect("Failed to create HTTP client")
 }
+
+#[allow(clippy::field_reassign_with_default)]
+pub async fn assign_response_data(http_data: &mut HttpData, resp: Response, return_filters: bool) {
+    let headers = resp.headers().clone();
+    let url = resp.url().clone();
+
+    http_data.http_status = "ACTIVE".to_string();
+    http_data.content_type = headers
+        .get(CONTENT_TYPE)
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or_default()
+        .to_string();
+
+    let full_body = {
+        const MAX_BODY_SIZE: usize = 1024 * 1024; // 1MB limit
+        match resp.text().await {
+            Ok(text) if text.len() <= MAX_BODY_SIZE => text,
+            Ok(text) => text.chars().take(MAX_BODY_SIZE).collect(),
+            Err(_) => String::new(),
+        }
+    };
+
+    http_data.content_length = headers
+        .get(CONTENT_LENGTH)
+        .and_then(|v| v.to_str().ok()?.parse().ok())
+        .unwrap_or_else(|| full_body.chars().count() as u64);
+
+    http_data.headers = format!("{headers:?}");
+
+    return_title_and_body(http_data, &full_body);
+
+    let lines_count = full_body.lines().count();
+    let words_count = full_body.split_whitespace().count();
+
+    http_data.words_count = words_count;
+    http_data.lines = lines_count + 1;
+    http_data.points_to_another_host = url.host_str() != Some(&http_data.checked_host);
+
+    if return_filters {
+        let host = url.host_str().unwrap_or_default();
+        let client = return_http_client(5, 3);
+        let user_agents = utils::user_agents();
+        http_data.bad_data = return_filters_data(host, client, user_agents).await;
+    }
+}
+
+pub fn return_title_and_body(http_data: &mut HttpData, body: &str) {
+    let document = Html::parse_document(body);
+
+    let title_selector = Selector::parse("title").ok();
+    let body_selector = Selector::parse("body").ok();
+
+    if let Some(title_sel) = &title_selector {
+        http_data.title = document
+            .select(title_sel)
+            .next()
+            .map(|element| element.inner_html())
+            .unwrap_or_else(|| "NULL".to_string());
+    }
+
+    if let Some(body_sel) = &body_selector {
+        http_data.body = document
+            .select(body_sel)
+            .next()
+            .map(|element| element.inner_html())
+            .unwrap_or_else(|| "NULL".to_string());
+    }
+}
+
+pub async fn return_filters_data(
+    host: &str,
+    client: Client,
+    user_agents_list: Vec<String>,
+) -> HTTPFilters {
+    let mut urls_to_check = HashSet::new();
+    let random_str = rng()
+        .sample_iter(Alphanumeric)
+        .take(16)
+        .map(char::from)
+        .collect::<String>();
+    let words = vec![
+        "admin".to_string() + &random_str + "/",
+        ".htaccess".to_string() + &random_str,
+        random_str.to_string() + "/",
+        random_str.to_string(),
+    ];
+    for word in words {
+        urls_to_check.insert(format!("{}/{}", &host, word));
+    }
+
+    let threads = urls_to_check.len();
+    let mut http_filters = HTTPFilters::default();
+
+    let lib_options = LibOptions {
+        hosts: urls_to_check,
+        client,
+        user_agents: user_agents_list,
+        retries: 1,
+        threads,
+        quiet_flag: true,
+        ..Default::default()
+    };
+
+    let data = return_http_data(&lib_options, false).await;
+
+    data.values().for_each(|http_data| {
+        http_filters
+            .bad_http_lengths
+            .push(http_data.content_length.to_string());
+        http_filters
+            .bad_words_numbers
+            .push(http_data.words_count.to_string());
+        http_filters
+            .bad_lines_numbers
+            .push(http_data.lines.to_string());
+    });
+
+    http_filters
+}

src/lib.rs

@@ -1,3 +1,4 @@
+pub mod args;
 pub mod httplib;
 pub mod structs;
 pub mod utils;

src/main.rs

@@ -1,143 +1,51 @@
-use std::collections::HashSet;
-
-use fhc::httplib;
-
 use {
-    clap::{value_t, App, Arg},
+    clap::Parser,
-    fhc::utils,
+    fhc::{args, httplib, structs::LibOptions, utils},
-    tokio::{
+    std::collections::HashSet,
-        self,
+    tokio::io::{self, AsyncReadExt},
-        io::{self, AsyncReadExt},
-    },
 };
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Eval args
-    let matches = App::new("FHC")
+    let args = args::Cli::parse();
-        .version(clap::crate_version!())
-        .author("Eduard Tolosa <edu4rdshl@protonmail.com>")
-        .about("Fast HTTP Checker.")
-        .arg(
-            Arg::with_name("threads")
-                .short("t")
-                .long("threads")
-                .takes_value(true)
-                .help("Number of threads. Default: 50"),
-        )
-        .arg(
-            Arg::with_name("timeout")
-                .long("timeout")
-                .takes_value(true)
-                .help("Timeout in seconds. Default: 3"),
-        )
-        .arg(
-            Arg::with_name("show-codes")
-                .short("s")
-                .long("show-codes")
-                .takes_value(false)
-                .help("Show status codes for discovered hosts."),
-        )
-        .arg(
-            Arg::with_name("domain")
-                .short("d")
-                .long("domain")
-                .takes_value(true)
-                .help("Target domain. When it's specified, a wordlist can be used from stdin for bruteforcing."),
-        )
-        .arg(
-            Arg::with_name("retries")
-                .short("r")
-                .long("retries")
-                .takes_value(true)
-                .help("Max number of http probes per target."),
-        )
-        .arg(
-            Arg::with_name("1xx")
-                .short("1")
-                .long("1xx")
-                .takes_value(false)
-                .help("Show URLs with 100-199 response codes only."),
-        )
-        .arg(
-            Arg::with_name("2xx")
-                .short("2")
-                .long("2xx")
-                .takes_value(false)
-                .help("Show URLs with 200-299 response codes only."),
-        )
-        .arg(
-            Arg::with_name("3xx")
-                .short("3")
-                .long("3xx")
-                .takes_value(false)
-                .help("Show URLs with 300-399 response codes only."),
-        )
-        .arg(
-            Arg::with_name("4xx")
-                .short("4")
-                .long("4xx")
-                .takes_value(false)
-                .help("Show URLs with 400-499 response codes only."),
-        )
-        .arg(
-            Arg::with_name("5xx")
-                .short("5")
-                .long("5xx")
-                .takes_value(false)
-                .help("Show URLs with 500-599 response codes only."),
-        )
-        .get_matches();
 
-    // Assign values or use defaults
-    let conditional_response_code = if matches.is_present("1xx") {
-        100
-    } else if matches.is_present("2xx") {
-        200
-    } else if matches.is_present("3xx") {
-        300
-    } else if matches.is_present("4xx") {
-        400
-    } else if matches.is_present("5xx") {
-        500
-    } else {
-        0
-    };
-    let threads = value_t!(matches.value_of("threads"), usize).unwrap_or_else(|_| 50);
-    let retries = value_t!(matches.value_of("retries"), usize).unwrap_or_else(|_| 1);
-    let timeout = value_t!(matches.value_of("timeout"), u64).unwrap_or_else(|_| 3);
-    let user_agents_list = utils::user_agents();
-    let show_status_codes = matches.is_present("show-codes");
-
-    let client = httplib::return_http_client(timeout);
-
-    // Read stdin
-
-    let mut stdin = io::stdin();
     let mut buffer = String::new();
-    stdin.read_to_string(&mut buffer).await?;
+    let mut hosts = HashSet::new();
 
-    let hosts: HashSet<String> = if matches.is_present("domain") {
+    if args.domain.is_some() {
-        let domain = value_t!(matches, "domain", String).unwrap();
+        let domain = args.domain.expect("Error getting domain");
-        buffer
+        if args.bruteforce {
-            .lines()
+            io::stdin().read_to_string(&mut buffer).await?;
-            .map(|word| format!("{}.{}", word, domain))
+            hosts = buffer
-            .collect()
+                .lines()
+                .map(|word| format!("{word}.{domain}"))
+                .collect();
+        } else {
+            hosts.insert(domain);
+        }
     } else {
-        buffer.lines().map(str::to_owned).collect()
+        io::stdin().read_to_string(&mut buffer).await?;
+        hosts = buffer.lines().map(str::to_owned).collect();
+    }
+
+    let lib_options = LibOptions {
+        hosts,
+        client: httplib::return_http_client(args.timeout, args.max_redirects),
+        user_agents: utils::user_agents(),
+        retries: args.retries,
+        threads: args.threads,
+        filter_codes: args.filter_codes,
+        exclude_codes: args.exclude_codes,
+        show_full_data: args.show_full_data,
+        ..Default::default()
     };
 
-    httplib::return_http_data(
+    if !args.quiet && args.show_full_data {
-        hosts,
+        println!("DOMAIN,[FINAL_URL],[STATUS_CODE]");
-        client,
+    }
-        user_agents_list,
+
-        retries,
+    let _ = httplib::return_http_data(&lib_options, true).await;
-        threads,
-        conditional_response_code,
-        show_status_codes,
-        false,
-    )
-    .await;
 
     Ok(())
 }

src/structs.rs

@@ -1,16 +1,53 @@
-#[derive(Clone)]
+use std::collections::HashSet;
-pub struct HttpData {
+
-    pub http_status: String,
+#[allow(clippy::upper_case_acronyms)]
-    pub host_url: String,
+#[derive(Clone, Debug, Eq, PartialEq, Hash, Default)]
-    pub status_code: u16,
+pub struct HTTPFilters {
+    pub bad_http_lengths: Vec<String>,
+    pub bad_words_numbers: Vec<String>,
+    pub bad_lines_numbers: Vec<String>,
 }
 
-impl Default for HttpData {
+#[derive(Clone, Debug, Eq, PartialEq, Hash, Default)]
-    fn default() -> Self {
+pub struct HttpData {
-        HttpData {
+    pub http_status: String,
-            http_status: String::from(""),
+    pub status_code: u16,
-            host_url: String::from(""),
+    pub checked_host: String,
-            status_code: 0,
+    pub final_url: String,
+    pub protocol: String,
+    pub title: String,
+    pub content_type: String,
+    pub body: String,
+    pub headers: String,
+    pub content_length: u64,
+    pub words_count: usize,
+    pub lines: usize,
+    pub bad_data: HTTPFilters,
+    pub html_file_path: String,
+    pub screenshot_data: Vec<u8>,
+    pub points_to_another_host: bool,
+}
+
+impl HttpData {
+    #[inline]
+    pub fn new(host: String) -> Self {
+        Self {
+            checked_host: host,
+            ..Default::default()
         }
     }
 }
+
+#[derive(Clone, Debug, Default)]
+pub struct LibOptions {
+    pub hosts: HashSet<String>,
+    pub client: reqwest::Client,
+    pub user_agents: Vec<String>,
+    pub retries: usize,
+    pub threads: usize,
+    pub return_filters: bool,
+    pub filter_codes: Option<String>,
+    pub exclude_codes: Option<String>,
+    pub show_full_data: bool,
+    pub quiet_flag: bool,
+}

src/utils.rs

@@ -1,5 +1,6 @@
-use rand::{seq::SliceRandom, thread_rng};
+use rand::{rng, seq::IndexedRandom};
 
+#[must_use]
 pub fn user_agents() -> Vec<String> {
     vec!["Mozilla/5.0 (Windows NT 6.1; WOW64; rv:77.0) Gecko/20190101 Firefox/77.0".to_string(),
     "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:77.0) Gecko/20100101 Firefox/77.0".to_string(),
@@ -32,10 +33,11 @@ pub fn user_agents() -> Vec<String> {
     ]
 }
 
-pub fn return_random_string(strings: &[String]) -> String {
+#[must_use]
-    if strings.is_empty() {
+pub fn return_random_user_agent(strings: &[String]) -> String {
-        String::new()
+    let empty_string = String::new();
-    } else {
+    strings
-        strings.choose(&mut thread_rng()).unwrap().to_string()
+        .choose(&mut rng())
-    }
+        .unwrap_or(&empty_string)
+        .to_string()
 }