Compare commits

..

16 commits
0.8.1 ... main

Author SHA1 Message Date
ed011be8a6 Update builder.sh 2025-08-02 13:55:33 -05:00
70ee3429ae Update manpage 2025-08-02 13:13:47 -05:00
7b39d6ebc8 Bump version 2025-08-02 13:11:36 -05:00
e7917b1e47 Bump version 2025-08-02 13:10:43 -05:00
06b99f9b5d (chore): update manpage 2025-08-02 12:53:54 -05:00
12b42d9a05 (chore): update builder.sh
- Now we use podman, so checking por docker service doesn't makes sense
- Use cross for everything, otherwise the build ends up in glibc versions mismatch
2025-08-02 12:53:33 -05:00
6701590efd (chore): bump version 2025-07-27 16:52:12 -05:00
cc37ccbcac (feat): refactor code.
This refactor is aimed to provide better readability for future maintenance, at the same time that provides several performance improvements by reducing the number of repeated calls and improving the host-alive detection logic.
2025-07-27 16:51:47 -05:00
fa58e6aa7e (chore): bump version 2025-07-27 12:13:36 -05:00
291f8dfe9e (chore): update dependencies and fix code to match them 2025-07-27 12:12:35 -05:00
99db1a4b29 Update dependencies. 2025-04-20 13:32:58 -05:00
c0ba951955 Clippy fixes 2025-04-20 13:32:54 -05:00
002a612e70 Update builder. 2025-04-20 13:32:40 -05:00
cdd2d778b4 Update manpage. 2024-10-28 11:33:05 -05:00
973c6fc1c9 Bump version. 2024-10-28 11:30:39 -05:00
9d55a00335 Disable redirects by default. See Probing issue! #9 2024-10-28 11:30:29 -05:00
9 changed files with 956 additions and 569 deletions

1327
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,6 @@
[package]
name = "fhc"
version = "0.8.1"
version = "0.9.1"
authors = ["Eduard Tolosa <edu4rdshl@protonmail.com>"]
edition = "2018"
description = "Fast HTTP Checker."
@ -12,22 +12,21 @@ readme = "README.md"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tokio = { version = "1.41.0", features = ["full", "io-util"] }
tokio = { version = "1.47.0", features = ["full", "io-util"] }
futures = "0.3.31"
clap = { version = "4.5.20", features = ["derive"] }
reqwest = { version = "0.12.8", features = [
clap = { version = "4.5.41", features = ["derive"] }
reqwest = { version = "0.12.22", features = [
"hickory-dns",
"rustls-tls",
"native-tls",
] }
openssl = { version = "0.10.68", features = ["vendored"] }
rand = "0.8.5"
scraper = "0.20.0"
openssl = { version = "0.10.73", features = ["vendored"] }
rand = "0.9.2"
scraper = "0.23.1"
async-recursion = "1.1.1"
[profile.release]
lto = 'thin'
lto = true
codegen-units = 1
panic = 'abort'
incremental = false
opt-level = "s"
strip = true

View file

@ -10,22 +10,17 @@ AARCH_TARGET="aarch64-unknown-linux-gnu"
MANPAGE_DIR="./$NAME.1"
BIN_OUTPUT_DIR="./ghbinaries"
if ! systemctl is-active docker >/dev/null 2>&1; then
echo "Docker is not running. Starting docker."
if ! sudo systemctl start docker; then
echo "Failed to start docker."
exit 1
fi
fi
rm -rf "$BIN_OUTPUT_DIR"
mkdir -p "$BIN_OUTPUT_DIR"
# Linux build
echo "Building Linux artifact."
if cargo build -q --release --target="$LINUX_TARGET"; then
if cross build -q --release --target="$LINUX_TARGET"; then
echo "Linux artifact build: SUCCESS"
cp "target/$LINUX_TARGET/release/$NAME" "target/$LINUX_TARGET/release/$NAME-linux"
strip "target/$LINUX_TARGET/release/$NAME-linux"
sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"target/$LINUX_TARGET/release/$NAME-linux.sha512"
zip -q "$BIN_OUTPUT_DIR/$NAME-linux.zip" "target/$LINUX_TARGET/release/$NAME-linux"
sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"$BIN_OUTPUT_DIR/$NAME-linux.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-x64.zip" "target/$LINUX_TARGET/release/$NAME-linux"
else
echo "Linux artifact build: FAILED"
fi
@ -36,8 +31,8 @@ if cross build -q --release --target="$LINUX_X86_TARGET"; then
echo "Linux x86 artifact build: SUCCESS"
cp "target/$LINUX_X86_TARGET/release/$NAME" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
strip "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"target/$LINUX_X86_TARGET/release/$NAME-linux-i386.sha512"
zip -q "$BIN_OUTPUT_DIR/$NAME-linux-i386.zip" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"$BIN_OUTPUT_DIR/$NAME-linux-i386.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-i386.zip" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
else
echo "Linux x86 artifact build: FAILED"
fi
@ -48,8 +43,8 @@ if cross build -q --release --target="$WIN_TARGET"; then
echo "Windows artifact build: SUCCESS"
cp "target/$WIN_TARGET/release/$NAME.exe" "target/$WIN_TARGET/release/$NAME-windows.exe"
strip "target/$WIN_TARGET/release/$NAME-windows.exe"
sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"target/$WIN_TARGET/release/$NAME-windows.exe.sha512"
zip -q "$BIN_OUTPUT_DIR/$NAME-windows.zip" "target/$WIN_TARGET/release/$NAME-windows.exe"
sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"$BIN_OUTPUT_DIR/$NAME-windows.exe.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-windows.zip" "target/$WIN_TARGET/release/$NAME-windows.exe"
else
echo "Windows artifact build: FAILED"
fi
@ -60,8 +55,8 @@ if cross build -q --release --target="$ARMV7_TARGET"; then
echo "ARMv7 artifact build: SUCCESS"
cp "target/$ARMV7_TARGET/release/$NAME" "target/$ARMV7_TARGET/release/$NAME-armv7"
strip "target/$ARMV7_TARGET/release/$NAME-armv7"
sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"target/$ARMV7_TARGET/release/$NAME-armv7.sha512"
zip -q "$BIN_OUTPUT_DIR/$NAME-armv7.zip" "target/$ARMV7_TARGET/release/$NAME-armv7"
sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"$BIN_OUTPUT_DIR/$NAME-armv7.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-armv7.zip" "target/$ARMV7_TARGET/release/$NAME-armv7"
else
echo "ARMv7 artifact build: FAILED"
fi
@ -72,8 +67,8 @@ if cross build -q --release --target="$AARCH_TARGET"; then
echo "Aarch64 artifact build: SUCCESS"
cp "target/$AARCH_TARGET/release/$NAME" "target/$AARCH_TARGET/release/$NAME-aarch64"
strip "target/$AARCH_TARGET/release/$NAME-aarch64"
sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"target/$AARCH_TARGET/release/$NAME-aarch64.sha512"
zip -q "$BIN_OUTPUT_DIR/$NAME-aarch64.zip" "target/$AARCH_TARGET/release/$NAME-aarch64"
sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"$BIN_OUTPUT_DIR/$NAME-aarch64.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-aarch64.zip" "target/$AARCH_TARGET/release/$NAME-aarch64"
else
echo "Aarch64 artifact build: FAILED"
fi

6
fhc.1
View file

@ -1,7 +1,7 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3.
.TH FHC "1" "October 2024" "fhc 0.8.1" "User Commands"
.TH FHC "1" "August 2025" "fhc 0.9.1" "User Commands"
.SH NAME
fhc \- manual page for fhc 0.8.1
fhc \- manual page for fhc 0.9.1
.SH SYNOPSIS
.B fhc
[\fI\,OPTIONS\/\fR]
@ -25,7 +25,7 @@ Domain to check \- can be omitted if using stdin
Number of retries [default: 1]
.TP
\fB\-L\fR, \fB\-\-max\-redirects\fR <MAX_REDIRECTS>
Maximum number of redirects [default: 10]
Maximum number of redirects, disabled by default [default: 0]
.TP
\fB\-b\fR, \fB\-\-bruteforce\fR
Enable bruteforce mode

View file

@ -18,8 +18,8 @@ pub struct Cli {
#[clap(short, long, default_value = "1")]
/// Number of retries
pub retries: usize,
#[clap(short = 'L', long, default_value = "10")]
/// Maximum number of redirects
#[clap(short = 'L', long, default_value = "0")]
/// Maximum number of redirects, disabled by default.
pub max_redirects: usize,
#[clap(short, long)]
/// Enable bruteforce mode

View file

@ -4,7 +4,7 @@ use {
utils,
},
futures::stream::StreamExt,
rand::{distributions::Alphanumeric, thread_rng as rng, Rng},
rand::{distr::Alphanumeric, rng, Rng},
reqwest::{
header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT},
redirect::Policy,
@ -15,37 +15,37 @@ use {
};
#[must_use]
#[allow(clippy::similar_names)]
pub async fn return_http_data(options: &LibOptions, from_cli: bool) -> HashMap<String, HttpData> {
let threads = options.hosts.len().min(options.threads);
let filter_codes = options.filter_codes.as_deref().unwrap_or_default();
let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default();
futures::stream::iter(options.hosts.clone().into_iter().map(|host| {
futures::stream::iter(options.hosts.iter().map(|host| {
let user_agent = utils::return_random_user_agent(&options.user_agents);
async move {
let mut http_data = HttpData {
checked_host: host.clone(),
..Default::default()
};
let mut http_data = HttpData::new(host.clone());
let mut response = None;
// Attempt both HTTPS and HTTP requests, retry if necessary
for _ in 0..options.retries {
let https_req = options
.client
.get(format!("https://{}", host))
.get(format!("https://{host}"))
.header(USER_AGENT, &user_agent)
.send();
let http_req = options
.client
.get(format!("http://{}", host))
.get(format!("http://{host}"))
.header(USER_AGENT, &user_agent)
.send();
response = https_req.await.or(http_req.await).ok();
response = tokio::select! {
https_result = https_req => https_result.ok(),
http_result = http_req => http_result.ok(),
};
if response.is_some() {
break;
@ -72,16 +72,26 @@ pub async fn return_http_data(options: &LibOptions, from_cli: bool) -> HashMap<S
&& (exclude_codes.is_empty()
|| !exclude_codes.contains(&http_data.status_code.to_string()))
{
// Use faster I/O for high-throughput scenarios
use std::io::{self, Write};
let stdout = io::stdout();
let mut handle = stdout.lock();
if options.show_full_data {
println!(
let _ = writeln!(
handle,
"{},[{}],[{}]",
http_data.checked_host, http_data.final_url, http_data.status_code
);
} else {
println!("{}://{}", http_data.protocol, http_data.checked_host);
let _ = writeln!(
handle,
"{}://{}",
http_data.protocol, http_data.checked_host
);
}
}
(host, http_data)
(host.clone(), http_data)
}
}))
.buffer_unordered(threads)
@ -102,6 +112,9 @@ pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client {
.redirect(policy)
.danger_accept_invalid_certs(true)
.use_native_tls()
.pool_max_idle_per_host(50)
.pool_idle_timeout(std::time::Duration::from_secs(30))
.tcp_keepalive(std::time::Duration::from_secs(60))
.build()
.expect("Failed to create HTTP client")
}
@ -118,7 +131,14 @@ pub async fn assign_response_data(http_data: &mut HttpData, resp: Response, retu
.unwrap_or_default()
.to_string();
let full_body = (resp.text().await).unwrap_or_default();
let full_body = {
const MAX_BODY_SIZE: usize = 1024 * 1024; // 1MB limit
match resp.text().await {
Ok(text) if text.len() <= MAX_BODY_SIZE => text,
Ok(text) => text.chars().take(MAX_BODY_SIZE).collect(),
Err(_) => String::new(),
}
};
http_data.content_length = headers
.get(CONTENT_LENGTH)
@ -129,8 +149,11 @@ pub async fn assign_response_data(http_data: &mut HttpData, resp: Response, retu
return_title_and_body(http_data, &full_body);
http_data.words_count = full_body.split_whitespace().count();
http_data.lines = full_body.lines().count() + 1;
let lines_count = full_body.lines().count();
let words_count = full_body.split_whitespace().count();
http_data.words_count = words_count;
http_data.lines = lines_count + 1;
http_data.points_to_another_host = url.host_str() != Some(&http_data.checked_host);
if return_filters {
@ -144,35 +167,24 @@ pub async fn assign_response_data(http_data: &mut HttpData, resp: Response, retu
pub fn return_title_and_body(http_data: &mut HttpData, body: &str) {
let document = Html::parse_document(body);
// Return title
match Selector::parse("title") {
Ok(selector) => {
if let Some(title_element) = document.select(&selector).next() {
http_data.title = title_element.inner_html();
} else {
http_data.title = "NULL".to_string();
}
}
Err(err) => {
eprintln!("Failed to parse selector: {err:?}");
}
let title_selector = Selector::parse("title").ok();
let body_selector = Selector::parse("body").ok();
if let Some(title_sel) = &title_selector {
http_data.title = document
.select(title_sel)
.next()
.map(|element| element.inner_html())
.unwrap_or_else(|| "NULL".to_string());
}
// Return body
match Selector::parse("body") {
Ok(selector) => {
if let Some(body_element) = document.select(&selector).next() {
http_data.body = body_element.inner_html();
} else {
http_data.body = "NULL".to_string();
}
}
Err(err) => {
eprintln!("Failed to parse selector: {err:?}");
}
if let Some(body_sel) = &body_selector {
http_data.body = document
.select(body_sel)
.next()
.map(|element| element.inner_html())
.unwrap_or_else(|| "NULL".to_string());
}
drop(document);
}
pub async fn return_filters_data(
@ -211,21 +223,17 @@ pub async fn return_filters_data(
let data = return_http_data(&lib_options, false).await;
data.values()
.map(|http_data| {
http_filters
.bad_http_lengths
.append(&mut vec![http_data.content_length.to_string()]);
http_filters.bad_words_numbers.append(&mut vec![http_data
.body
.split(' ')
.count()
.to_string()]);
http_filters
.bad_lines_numbers
.append(&mut vec![http_data.lines.to_string()]);
})
.for_each(drop);
data.values().for_each(|http_data| {
http_filters
.bad_http_lengths
.push(http_data.content_length.to_string());
http_filters
.bad_words_numbers
.push(http_data.words_count.to_string());
http_filters
.bad_lines_numbers
.push(http_data.lines.to_string());
});
http_filters
}

View file

@ -27,7 +27,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
} else {
io::stdin().read_to_string(&mut buffer).await?;
hosts = buffer.lines().map(str::to_owned).collect();
};
}
let lib_options = LibOptions {
hosts,

View file

@ -28,6 +28,16 @@ pub struct HttpData {
pub points_to_another_host: bool,
}
impl HttpData {
#[inline]
pub fn new(host: String) -> Self {
Self {
checked_host: host,
..Default::default()
}
}
}
#[derive(Clone, Debug, Default)]
pub struct LibOptions {
pub hosts: HashSet<String>,

View file

@ -1,4 +1,4 @@
use rand::{seq::SliceRandom, thread_rng};
use rand::{rng, seq::IndexedRandom};
#[must_use]
pub fn user_agents() -> Vec<String> {
@ -37,7 +37,7 @@ pub fn user_agents() -> Vec<String> {
pub fn return_random_user_agent(strings: &[String]) -> String {
let empty_string = String::new();
strings
.choose(&mut thread_rng())
.choose(&mut rng())
.unwrap_or(&empty_string)
.to_string()
}