Compare commits

..

20 commits
0.8.0 ... main

Author SHA1 Message Date
ed011be8a6 Update builder.sh 2025-08-02 13:55:33 -05:00
70ee3429ae Update manpage 2025-08-02 13:13:47 -05:00
7b39d6ebc8 Bump version 2025-08-02 13:11:36 -05:00
e7917b1e47 Bump version 2025-08-02 13:10:43 -05:00
06b99f9b5d (chore): update manpage 2025-08-02 12:53:54 -05:00
12b42d9a05 (chore): update builder.sh
- Now we use podman, so checking por docker service doesn't makes sense
- Use cross for everything, otherwise the build ends up in glibc versions mismatch
2025-08-02 12:53:33 -05:00
6701590efd (chore): bump version 2025-07-27 16:52:12 -05:00
cc37ccbcac (feat): refactor code.
This refactor is aimed to provide better readability for future maintenance, at the same time that provides several performance improvements by reducing the number of repeated calls and improving the host-alive detection logic.
2025-07-27 16:51:47 -05:00
fa58e6aa7e (chore): bump version 2025-07-27 12:13:36 -05:00
291f8dfe9e (chore): update dependencies and fix code to match them 2025-07-27 12:12:35 -05:00
99db1a4b29 Update dependencies. 2025-04-20 13:32:58 -05:00
c0ba951955 Clippy fixes 2025-04-20 13:32:54 -05:00
002a612e70 Update builder. 2025-04-20 13:32:40 -05:00
cdd2d778b4 Update manpage. 2024-10-28 11:33:05 -05:00
973c6fc1c9 Bump version. 2024-10-28 11:30:39 -05:00
9d55a00335 Disable redirects by default. See Probing issue! #9 2024-10-28 11:30:29 -05:00
200eac74dd Bump version. 2024-10-28 02:30:52 -05:00
ad731731b7 Performance improvements. 2024-10-27 16:26:08 -05:00
ede2c96e0f Update dependencies. 2024-10-27 16:24:21 -05:00
3a0674997e Update builder.sh 2024-10-27 16:24:06 -05:00
10 changed files with 1034 additions and 676 deletions

1
.gitignore vendored
View file

@ -2,3 +2,4 @@
/hosts.txt /hosts.txt
*.txt *.txt
armbuilder.sh armbuilder.sh
/ghbinaries

1335
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,6 @@
[package] [package]
name = "fhc" name = "fhc"
version = "0.8.0" version = "0.9.1"
authors = ["Eduard Tolosa <edu4rdshl@protonmail.com>"] authors = ["Eduard Tolosa <edu4rdshl@protonmail.com>"]
edition = "2018" edition = "2018"
description = "Fast HTTP Checker." description = "Fast HTTP Checker."
@ -12,22 +12,21 @@ readme = "README.md"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
tokio = { version = "1.40.0", features = ["full", "io-util"] } tokio = { version = "1.47.0", features = ["full", "io-util"] }
futures = "0.3.31" futures = "0.3.31"
clap = { version = "4.5.20", features = ["derive"] } clap = { version = "4.5.41", features = ["derive"] }
reqwest = { version = "0.12.8", features = [ reqwest = { version = "0.12.22", features = [
"hickory-dns", "hickory-dns",
"rustls-tls", "rustls-tls",
"native-tls", "native-tls",
] } ] }
openssl = { version = "0.10.66", features = ["vendored"] } openssl = { version = "0.10.73", features = ["vendored"] }
rand = "0.8.5" rand = "0.9.2"
scraper = "0.20.0" scraper = "0.23.1"
async-recursion = "1.1.1" async-recursion = "1.1.1"
[profile.release] [profile.release]
lto = 'thin' lto = true
codegen-units = 1 codegen-units = 1
panic = 'abort' panic = 'abort'
incremental = false strip = true
opt-level = "s"

View file

@ -6,24 +6,21 @@ LINUX_X86_TARGET="i686-unknown-linux-musl"
WIN_TARGET="x86_64-pc-windows-gnu" WIN_TARGET="x86_64-pc-windows-gnu"
ARMV7_TARGET="armv7-unknown-linux-gnueabihf" ARMV7_TARGET="armv7-unknown-linux-gnueabihf"
AARCH_TARGET="aarch64-unknown-linux-gnu" AARCH_TARGET="aarch64-unknown-linux-gnu"
OSX_TARGET="x86_64-apple-darwin" # OSX_TARGET="x86_64-apple-darwin"
MANPAGE_DIR="./$NAME.1" MANPAGE_DIR="./$NAME.1"
BIN_OUTPUT_DIR="./ghbinaries"
if ! systemctl is-active docker >/dev/null 2>&1; then rm -rf "$BIN_OUTPUT_DIR"
echo "Docker is not running. Starting docker." mkdir -p "$BIN_OUTPUT_DIR"
if ! sudo systemctl start docker; then
echo "Failed to start docker."
exit 1
fi
fi
# Linux build # Linux build
echo "Building Linux artifact." echo "Building Linux artifact."
if cargo build -q --release --target="$LINUX_TARGET"; then if cross build -q --release --target="$LINUX_TARGET"; then
echo "Linux artifact build: SUCCESS" echo "Linux artifact build: SUCCESS"
cp "target/$LINUX_TARGET/release/$NAME" "target/$LINUX_TARGET/release/$NAME-linux" cp "target/$LINUX_TARGET/release/$NAME" "target/$LINUX_TARGET/release/$NAME-linux"
strip "target/$LINUX_TARGET/release/$NAME-linux" strip "target/$LINUX_TARGET/release/$NAME-linux"
sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"target/$LINUX_TARGET/release/$NAME-linux.sha512" sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"$BIN_OUTPUT_DIR/$NAME-linux.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-x64.zip" "target/$LINUX_TARGET/release/$NAME-linux"
else else
echo "Linux artifact build: FAILED" echo "Linux artifact build: FAILED"
fi fi
@ -34,7 +31,8 @@ if cross build -q --release --target="$LINUX_X86_TARGET"; then
echo "Linux x86 artifact build: SUCCESS" echo "Linux x86 artifact build: SUCCESS"
cp "target/$LINUX_X86_TARGET/release/$NAME" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" cp "target/$LINUX_X86_TARGET/release/$NAME" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
strip "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" strip "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"target/$LINUX_X86_TARGET/release/$NAME-linux-i386.sha512" sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"$BIN_OUTPUT_DIR/$NAME-linux-i386.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-i386.zip" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
else else
echo "Linux x86 artifact build: FAILED" echo "Linux x86 artifact build: FAILED"
fi fi
@ -45,7 +43,8 @@ if cross build -q --release --target="$WIN_TARGET"; then
echo "Windows artifact build: SUCCESS" echo "Windows artifact build: SUCCESS"
cp "target/$WIN_TARGET/release/$NAME.exe" "target/$WIN_TARGET/release/$NAME-windows.exe" cp "target/$WIN_TARGET/release/$NAME.exe" "target/$WIN_TARGET/release/$NAME-windows.exe"
strip "target/$WIN_TARGET/release/$NAME-windows.exe" strip "target/$WIN_TARGET/release/$NAME-windows.exe"
sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"target/$WIN_TARGET/release/$NAME-windows.exe.sha512" sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"$BIN_OUTPUT_DIR/$NAME-windows.exe.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-windows.zip" "target/$WIN_TARGET/release/$NAME-windows.exe"
else else
echo "Windows artifact build: FAILED" echo "Windows artifact build: FAILED"
fi fi
@ -56,7 +55,8 @@ if cross build -q --release --target="$ARMV7_TARGET"; then
echo "ARMv7 artifact build: SUCCESS" echo "ARMv7 artifact build: SUCCESS"
cp "target/$ARMV7_TARGET/release/$NAME" "target/$ARMV7_TARGET/release/$NAME-armv7" cp "target/$ARMV7_TARGET/release/$NAME" "target/$ARMV7_TARGET/release/$NAME-armv7"
strip "target/$ARMV7_TARGET/release/$NAME-armv7" strip "target/$ARMV7_TARGET/release/$NAME-armv7"
sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"target/$ARMV7_TARGET/release/$NAME-armv7.sha512" sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"$BIN_OUTPUT_DIR/$NAME-armv7.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-armv7.zip" "target/$ARMV7_TARGET/release/$NAME-armv7"
else else
echo "ARMv7 artifact build: FAILED" echo "ARMv7 artifact build: FAILED"
fi fi
@ -67,21 +67,23 @@ if cross build -q --release --target="$AARCH_TARGET"; then
echo "Aarch64 artifact build: SUCCESS" echo "Aarch64 artifact build: SUCCESS"
cp "target/$AARCH_TARGET/release/$NAME" "target/$AARCH_TARGET/release/$NAME-aarch64" cp "target/$AARCH_TARGET/release/$NAME" "target/$AARCH_TARGET/release/$NAME-aarch64"
strip "target/$AARCH_TARGET/release/$NAME-aarch64" strip "target/$AARCH_TARGET/release/$NAME-aarch64"
sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"target/$AARCH_TARGET/release/$NAME-aarch64.sha512" sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"$BIN_OUTPUT_DIR/$NAME-aarch64.sha512"
zip -q -j "$BIN_OUTPUT_DIR/$NAME-aarch64.zip" "target/$AARCH_TARGET/release/$NAME-aarch64"
else else
echo "Aarch64 artifact build: FAILED" echo "Aarch64 artifact build: FAILED"
fi fi
# Mac OS build # # Mac OS build
echo "Building OSX artifact." # echo "Building OSX artifact."
if CC=o64-clang CXX=o64-clang++ LIBZ_SYS_STATIC=1 cargo build -q --release --target="$OSX_TARGET"; then # if CC=o64-clang CXX=o64-clang++ LIBZ_SYS_STATIC=1 cargo build -q --release --target="$OSX_TARGET"; then
echo "OSX artifact build: SUCCESS" # echo "OSX artifact build: SUCCESS"
cp "target/$OSX_TARGET/release/$NAME" "target/$OSX_TARGET/release/$NAME-osx" # cp "target/$OSX_TARGET/release/$NAME" "target/$OSX_TARGET/release/$NAME-osx"
strip "target/$OSX_TARGET/release/$NAME-osx" # strip "target/$OSX_TARGET/release/$NAME-osx"
sha512sum "target/$OSX_TARGET/release/$NAME-osx" >"target/$OSX_TARGET/release/$NAME-osx.sha512" # sha512sum "target/$OSX_TARGET/release/$NAME-osx" >"target/$OSX_TARGET/release/$NAME-osx.sha512"
else # zip -q "$BIN_OUTPUT_DIR/$NAME-osx.zip" "target/$OSX_TARGET/release/$NAME-osx"
echo "OSX artifact build: FAILED" # else
fi # echo "OSX artifact build: FAILED"
# fi
echo "Creating manpage..." echo "Creating manpage..."
if command -v help2man >/dev/null; then if command -v help2man >/dev/null; then
@ -93,10 +95,3 @@ if command -v help2man >/dev/null; then
else else
echo "Please install the help2man package." echo "Please install the help2man package."
fi fi
# Stop docker
echo "Stopping docker."
if ! sudo systemctl stop docker; then
echo "Failed to stop docker."
exit 1
fi

81
fhc.1
View file

@ -1,65 +1,58 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.1. .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3.
.TH FHC "1" "April 2022" "FHC 0.7.1" "User Commands" .TH FHC "1" "August 2025" "fhc 0.9.1" "User Commands"
.SH NAME .SH NAME
FHC \- manual page for FHC 0.7.1 fhc \- manual page for fhc 0.9.1
.SH SYNOPSIS
.B fhc
[\fI\,OPTIONS\/\fR]
.SH DESCRIPTION .SH DESCRIPTION
FHC 0.7.1
Eduard Tolosa <edu4rdshl@protonmail.com>
Fast HTTP Checker. Fast HTTP Checker.
.SS "USAGE:" .SH OPTIONS
.IP
fhc [FLAGS] [OPTIONS]
.SS "FLAGS:"
.TP .TP
\fB\-1\fR, \fB\-\-1xx\fR \fB\-t\fR, \fB\-\-threads\fR <THREADS>
Show URLs with 100\-199 response codes only. Number of threads to use [default: 50]
.TP .TP
\fB\-2\fR, \fB\-\-2xx\fR \fB\-\-timeout\fR <TIMEOUT>
Show URLs with 200\-299 response codes only. Timeout in seconds [default: 3]
.TP .TP
\fB\-3\fR, \fB\-\-3xx\fR \fB\-s\fR, \fB\-\-show\-full\-data\fR
Show URLs with 300\-399 response codes only. Show HTTP status codes, final URL and domain
.TP .TP
\fB\-4\fR, \fB\-\-4xx\fR \fB\-d\fR, \fB\-\-domain\fR <DOMAIN>
Show URLs with 400\-499 response codes only. Domain to check \- can be omitted if using stdin
.TP .TP
\fB\-5\fR, \fB\-\-5xx\fR \fB\-r\fR, \fB\-\-retries\fR <RETRIES>
Show URLs with 500\-599 response codes only. Number of retries [default: 1]
.TP
\fB\-L\fR, \fB\-\-max\-redirects\fR <MAX_REDIRECTS>
Maximum number of redirects, disabled by default [default: 0]
.TP
\fB\-b\fR, \fB\-\-bruteforce\fR
Enable bruteforce mode
.TP
\fB\-f\fR, \fB\-\-filter\-codes\fR <FILTER_CODES>
Filter status codes. A comma separated list can be used
.TP
\fB\-e\fR, \fB\-\-exclude\-codes\fR <EXCLUDE_CODES>
Exclude status codes. A comma separated list can be used
.TP
\fB\-q\fR, \fB\-\-quiet\fR
Quiet mode. This will suppress all fancy output except for the final results
.TP .TP
\fB\-h\fR, \fB\-\-help\fR \fB\-h\fR, \fB\-\-help\fR
Prints help information Print help
.TP
\fB\-s\fR, \fB\-\-show\-codes\fR
Show status codes for discovered hosts.
.TP .TP
\fB\-V\fR, \fB\-\-version\fR \fB\-V\fR, \fB\-\-version\fR
Prints version information Print version
.SS "OPTIONS:"
.TP
\fB\-d\fR, \fB\-\-domain\fR <domain>
Target domain. When it's specified, a wordlist can be used from stdin for
bruteforcing.
.TP
\fB\-L\fR, \fB\-\-max\-redirects\fR <max\-redirects>
Max number of redirects. Default: 0
.TP
\fB\-r\fR, \fB\-\-retries\fR <retries>
Max number of http probes per target.
.TP
\fB\-t\fR, \fB\-\-threads\fR <threads>
Number of threads. Default: 50
.TP
\fB\-\-timeout\fR <timeout>
Timeout in seconds. Default: 3
.SH "SEE ALSO" .SH "SEE ALSO"
The full documentation for The full documentation for
.B FHC .B fhc
is maintained as a Texinfo manual. If the is maintained as a Texinfo manual. If the
.B info .B info
and and
.B FHC .B fhc
programs are properly installed at your site, the command programs are properly installed at your site, the command
.IP .IP
.B info FHC .B info fhc
.PP .PP
should give you access to the complete manual. should give you access to the complete manual.

View file

@ -10,16 +10,16 @@ pub struct Cli {
/// Timeout in seconds /// Timeout in seconds
pub timeout: u64, pub timeout: u64,
#[clap(short, long)] #[clap(short, long)]
/// Show HTTP status codes /// Show HTTP status codes, final URL and domain
pub show_codes: bool, pub show_full_data: bool,
#[clap(short, long)] #[clap(short, long)]
/// Domain to check - can be omitted if using stdin /// Domain to check - can be omitted if using stdin
pub domain: Option<String>, pub domain: Option<String>,
#[clap(short, long, default_value = "1")] #[clap(short, long, default_value = "1")]
/// Number of retries /// Number of retries
pub retries: usize, pub retries: usize,
#[clap(short = 'L', long, default_value = "10")] #[clap(short = 'L', long, default_value = "0")]
/// Maximum number of redirects /// Maximum number of redirects, disabled by default.
pub max_redirects: usize, pub max_redirects: usize,
#[clap(short, long)] #[clap(short, long)]
/// Enable bruteforce mode /// Enable bruteforce mode

View file

@ -4,7 +4,7 @@ use {
utils, utils,
}, },
futures::stream::StreamExt, futures::stream::StreamExt,
rand::{distributions::Alphanumeric, thread_rng as rng, Rng}, rand::{distr::Alphanumeric, rng, Rng},
reqwest::{ reqwest::{
header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT}, header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT},
redirect::Policy, redirect::Policy,
@ -15,34 +15,37 @@ use {
}; };
#[must_use] #[must_use]
pub async fn return_http_data(options: &LibOptions) -> HashMap<String, HttpData> { #[allow(clippy::similar_names)]
pub async fn return_http_data(options: &LibOptions, from_cli: bool) -> HashMap<String, HttpData> {
let threads = options.hosts.len().min(options.threads); let threads = options.hosts.len().min(options.threads);
let filter_codes = options.filter_codes.as_deref().unwrap_or_default(); let filter_codes = options.filter_codes.as_deref().unwrap_or_default();
let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default(); let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default();
futures::stream::iter(options.hosts.clone().into_iter().map(|host| { futures::stream::iter(options.hosts.iter().map(|host| {
let user_agent = utils::return_random_user_agent(&options.user_agents); let user_agent = utils::return_random_user_agent(&options.user_agents);
async move { async move {
let mut http_data = HttpData::default(); let mut http_data = HttpData::new(host.clone());
let mut response = None; let mut response = None;
// Attempt both HTTPS and HTTP requests, retry if necessary
for _ in 0..options.retries { for _ in 0..options.retries {
let https_req = options let https_req = options
.client .client
.get(format!("https://{}", host)) .get(format!("https://{host}"))
.header(USER_AGENT, &user_agent) .header(USER_AGENT, &user_agent)
.send(); .send();
let http_req = options let http_req = options
.client .client
.get(format!("http://{}", host)) .get(format!("http://{host}"))
.header(USER_AGENT, &user_agent) .header(USER_AGENT, &user_agent)
.send(); .send();
response = https_req.await.or(http_req.await).ok(); response = tokio::select! {
https_result = https_req => https_result.ok(),
http_result = http_req => http_result.ok(),
};
if response.is_some() { if response.is_some() {
break; break;
@ -50,36 +53,45 @@ pub async fn return_http_data(options: &LibOptions) -> HashMap<String, HttpData>
} }
if let Some(resp) = response { if let Some(resp) = response {
http_data.checked_host = host.clone(); // Those are always set
http_data.protocol = resp.url().scheme().to_string();
http_data.status_code = resp.status().as_u16();
http_data.final_url = resp.url().to_string(); http_data.final_url = resp.url().to_string();
if options.assign_response_data { if !from_cli {
http_data = assign_response_data(http_data, resp, options.return_filters).await; assign_response_data(&mut http_data, resp, options.return_filters).await;
} else {
http_data.status_code = resp.status().as_u16();
http_data.http_status = "ACTIVE".to_string();
} }
} else { } else {
http_data.http_status = "INACTIVE".to_string(); http_data.http_status = "INACTIVE".to_string();
} }
if !options.quiet_flag if !options.quiet_flag
&& !http_data.checked_host.is_empty() && !http_data.final_url.is_empty()
&& (filter_codes.is_empty() && (filter_codes.is_empty()
|| filter_codes.contains(&http_data.status_code.to_string())) || filter_codes.contains(&http_data.status_code.to_string()))
&& (exclude_codes.is_empty() && (exclude_codes.is_empty()
|| !exclude_codes.contains(&http_data.status_code.to_string())) || !exclude_codes.contains(&http_data.status_code.to_string()))
{ {
if options.show_status_codes { // Use faster I/O for high-throughput scenarios
println!( use std::io::{self, Write};
let stdout = io::stdout();
let mut handle = stdout.lock();
if options.show_full_data {
let _ = writeln!(
handle,
"{},[{}],[{}]", "{},[{}],[{}]",
http_data.checked_host, http_data.final_url, http_data.status_code http_data.checked_host, http_data.final_url, http_data.status_code
); );
} else { } else {
println!("{},[{}]", http_data.checked_host, http_data.final_url); let _ = writeln!(
handle,
"{}://{}",
http_data.protocol, http_data.checked_host
);
} }
} }
(host, http_data) (host.clone(), http_data)
} }
})) }))
.buffer_unordered(threads) .buffer_unordered(threads)
@ -100,30 +112,33 @@ pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client {
.redirect(policy) .redirect(policy)
.danger_accept_invalid_certs(true) .danger_accept_invalid_certs(true)
.use_native_tls() .use_native_tls()
.pool_max_idle_per_host(50)
.pool_idle_timeout(std::time::Duration::from_secs(30))
.tcp_keepalive(std::time::Duration::from_secs(60))
.build() .build()
.expect("Failed to create HTTP client") .expect("Failed to create HTTP client")
} }
#[allow(clippy::field_reassign_with_default)] #[allow(clippy::field_reassign_with_default)]
pub async fn assign_response_data( pub async fn assign_response_data(http_data: &mut HttpData, resp: Response, return_filters: bool) {
mut http_data: HttpData,
resp: Response,
return_filters: bool,
) -> HttpData {
let headers = resp.headers().clone(); let headers = resp.headers().clone();
let url = resp.url().clone(); let url = resp.url().clone();
http_data.http_status = "ACTIVE".to_string(); http_data.http_status = "ACTIVE".to_string();
http_data.status_code = resp.status().as_u16();
http_data.final_url = url.to_string();
http_data.protocol = url.scheme().to_string();
http_data.content_type = headers http_data.content_type = headers
.get(CONTENT_TYPE) .get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok()) .and_then(|v| v.to_str().ok())
.unwrap_or_default() .unwrap_or_default()
.to_string(); .to_string();
let full_body = (resp.text().await).unwrap_or_default(); let full_body = {
const MAX_BODY_SIZE: usize = 1024 * 1024; // 1MB limit
match resp.text().await {
Ok(text) if text.len() <= MAX_BODY_SIZE => text,
Ok(text) => text.chars().take(MAX_BODY_SIZE).collect(),
Err(_) => String::new(),
}
};
http_data.content_length = headers http_data.content_length = headers
.get(CONTENT_LENGTH) .get(CONTENT_LENGTH)
@ -132,10 +147,14 @@ pub async fn assign_response_data(
http_data.headers = format!("{headers:?}"); http_data.headers = format!("{headers:?}");
return_title_and_body(&mut http_data, &full_body); return_title_and_body(http_data, &full_body);
http_data.words_count = full_body.split_whitespace().count(); let lines_count = full_body.lines().count();
http_data.lines = full_body.lines().count() + 1; let words_count = full_body.split_whitespace().count();
http_data.words_count = words_count;
http_data.lines = lines_count + 1;
http_data.points_to_another_host = url.host_str() != Some(&http_data.checked_host);
if return_filters { if return_filters {
let host = url.host_str().unwrap_or_default(); let host = url.host_str().unwrap_or_default();
@ -143,42 +162,29 @@ pub async fn assign_response_data(
let user_agents = utils::user_agents(); let user_agents = utils::user_agents();
http_data.bad_data = return_filters_data(host, client, user_agents).await; http_data.bad_data = return_filters_data(host, client, user_agents).await;
} }
http_data
} }
pub fn return_title_and_body(http_data: &mut HttpData, body: &str) { pub fn return_title_and_body(http_data: &mut HttpData, body: &str) {
let document = Html::parse_document(body); let document = Html::parse_document(body);
// Return title let title_selector = Selector::parse("title").ok();
match Selector::parse("title") { let body_selector = Selector::parse("body").ok();
Ok(selector) => {
if let Some(title_element) = document.select(&selector).next() { if let Some(title_sel) = &title_selector {
http_data.title = title_element.inner_html(); http_data.title = document
} else { .select(title_sel)
http_data.title = "NULL".to_string(); .next()
} .map(|element| element.inner_html())
} .unwrap_or_else(|| "NULL".to_string());
Err(err) => {
eprintln!("Failed to parse selector: {err:?}");
}
} }
// Return body if let Some(body_sel) = &body_selector {
match Selector::parse("body") { http_data.body = document
Ok(selector) => { .select(body_sel)
if let Some(body_element) = document.select(&selector).next() { .next()
http_data.body = body_element.inner_html(); .map(|element| element.inner_html())
} else { .unwrap_or_else(|| "NULL".to_string());
http_data.body = "NULL".to_string();
}
}
Err(err) => {
eprintln!("Failed to parse selector: {err:?}");
}
} }
drop(document);
} }
pub async fn return_filters_data( pub async fn return_filters_data(
@ -211,28 +217,23 @@ pub async fn return_filters_data(
user_agents: user_agents_list, user_agents: user_agents_list,
retries: 1, retries: 1,
threads, threads,
assign_response_data: true,
quiet_flag: true, quiet_flag: true,
..Default::default() ..Default::default()
}; };
let data = return_http_data(&lib_options).await; let data = return_http_data(&lib_options, false).await;
data.values() data.values().for_each(|http_data| {
.map(|http_data| { http_filters
http_filters .bad_http_lengths
.bad_http_lengths .push(http_data.content_length.to_string());
.append(&mut vec![http_data.content_length.to_string()]); http_filters
http_filters.bad_words_numbers.append(&mut vec![http_data .bad_words_numbers
.body .push(http_data.words_count.to_string());
.split(' ') http_filters
.count() .bad_lines_numbers
.to_string()]); .push(http_data.lines.to_string());
http_filters });
.bad_lines_numbers
.append(&mut vec![http_data.lines.to_string()]);
})
.for_each(drop);
http_filters http_filters
} }

View file

@ -10,19 +10,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Eval args // Eval args
let args = args::Cli::parse(); let args = args::Cli::parse();
let filter_codes = args.filter_codes;
let exclude_codes = args.exclude_codes;
let threads = args.threads;
let retries = args.retries;
let timeout = args.timeout;
let max_redirects = args.max_redirects;
let user_agents_list = utils::user_agents();
let show_status_codes = args.show_codes;
let client = httplib::return_http_client(timeout, max_redirects);
let mut buffer = String::new(); let mut buffer = String::new();
let mut hosts = HashSet::new(); let mut hosts = HashSet::new();
if args.domain.is_some() { if args.domain.is_some() {
@ -39,29 +27,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
} else { } else {
io::stdin().read_to_string(&mut buffer).await?; io::stdin().read_to_string(&mut buffer).await?;
hosts = buffer.lines().map(str::to_owned).collect(); hosts = buffer.lines().map(str::to_owned).collect();
}; }
let lib_options = LibOptions { let lib_options = LibOptions {
hosts, hosts,
client, client: httplib::return_http_client(args.timeout, args.max_redirects),
user_agents: user_agents_list, user_agents: utils::user_agents(),
retries, retries: args.retries,
threads, threads: args.threads,
filter_codes, filter_codes: args.filter_codes,
exclude_codes, exclude_codes: args.exclude_codes,
show_status_codes, show_full_data: args.show_full_data,
..Default::default() ..Default::default()
}; };
if !args.quiet { if !args.quiet && args.show_full_data {
if show_status_codes { println!("DOMAIN,[FINAL_URL],[STATUS_CODE]");
println!("DOMAIN,[FINAL_URL],[STATUS_CODE]");
} else {
println!("DOMAIN,[FINAL_URL]");
}
} }
let _ = httplib::return_http_data(&lib_options).await; let _ = httplib::return_http_data(&lib_options, true).await;
Ok(()) Ok(())
} }

View file

@ -25,6 +25,17 @@ pub struct HttpData {
pub bad_data: HTTPFilters, pub bad_data: HTTPFilters,
pub html_file_path: String, pub html_file_path: String,
pub screenshot_data: Vec<u8>, pub screenshot_data: Vec<u8>,
pub points_to_another_host: bool,
}
impl HttpData {
#[inline]
pub fn new(host: String) -> Self {
Self {
checked_host: host,
..Default::default()
}
}
} }
#[derive(Clone, Debug, Default)] #[derive(Clone, Debug, Default)]
@ -37,7 +48,6 @@ pub struct LibOptions {
pub return_filters: bool, pub return_filters: bool,
pub filter_codes: Option<String>, pub filter_codes: Option<String>,
pub exclude_codes: Option<String>, pub exclude_codes: Option<String>,
pub show_status_codes: bool, pub show_full_data: bool,
pub assign_response_data: bool,
pub quiet_flag: bool, pub quiet_flag: bool,
} }

View file

@ -1,4 +1,4 @@
use rand::{seq::SliceRandom, thread_rng}; use rand::{rng, seq::IndexedRandom};
#[must_use] #[must_use]
pub fn user_agents() -> Vec<String> { pub fn user_agents() -> Vec<String> {
@ -37,7 +37,7 @@ pub fn user_agents() -> Vec<String> {
pub fn return_random_user_agent(strings: &[String]) -> String { pub fn return_random_user_agent(strings: &[String]) -> String {
let empty_string = String::new(); let empty_string = String::new();
strings strings
.choose(&mut thread_rng()) .choose(&mut rng())
.unwrap_or(&empty_string) .unwrap_or(&empty_string)
.to_string() .to_string()
} }