mirror of
https://github.com/edu4rdshl/fhc.git
synced 2026-07-18 07:34:52 +00:00
Compare commits
20 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ed011be8a6 | |||
| 70ee3429ae | |||
| 7b39d6ebc8 | |||
| e7917b1e47 | |||
| 06b99f9b5d | |||
| 12b42d9a05 | |||
| 6701590efd | |||
| cc37ccbcac | |||
| fa58e6aa7e | |||
| 291f8dfe9e | |||
| 99db1a4b29 | |||
| c0ba951955 | |||
| 002a612e70 | |||
| cdd2d778b4 | |||
| 973c6fc1c9 | |||
| 9d55a00335 | |||
| 200eac74dd | |||
| ad731731b7 | |||
| ede2c96e0f | |||
| 3a0674997e |
10 changed files with 1034 additions and 676 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -2,3 +2,4 @@
|
||||||
/hosts.txt
|
/hosts.txt
|
||||||
*.txt
|
*.txt
|
||||||
armbuilder.sh
|
armbuilder.sh
|
||||||
|
/ghbinaries
|
||||||
|
|
|
||||||
1335
Cargo.lock
generated
1335
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
19
Cargo.toml
19
Cargo.toml
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "fhc"
|
name = "fhc"
|
||||||
version = "0.8.0"
|
version = "0.9.1"
|
||||||
authors = ["Eduard Tolosa <edu4rdshl@protonmail.com>"]
|
authors = ["Eduard Tolosa <edu4rdshl@protonmail.com>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
description = "Fast HTTP Checker."
|
description = "Fast HTTP Checker."
|
||||||
|
|
@ -12,22 +12,21 @@ readme = "README.md"
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
tokio = { version = "1.40.0", features = ["full", "io-util"] }
|
tokio = { version = "1.47.0", features = ["full", "io-util"] }
|
||||||
futures = "0.3.31"
|
futures = "0.3.31"
|
||||||
clap = { version = "4.5.20", features = ["derive"] }
|
clap = { version = "4.5.41", features = ["derive"] }
|
||||||
reqwest = { version = "0.12.8", features = [
|
reqwest = { version = "0.12.22", features = [
|
||||||
"hickory-dns",
|
"hickory-dns",
|
||||||
"rustls-tls",
|
"rustls-tls",
|
||||||
"native-tls",
|
"native-tls",
|
||||||
] }
|
] }
|
||||||
openssl = { version = "0.10.66", features = ["vendored"] }
|
openssl = { version = "0.10.73", features = ["vendored"] }
|
||||||
rand = "0.8.5"
|
rand = "0.9.2"
|
||||||
scraper = "0.20.0"
|
scraper = "0.23.1"
|
||||||
async-recursion = "1.1.1"
|
async-recursion = "1.1.1"
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
lto = 'thin'
|
lto = true
|
||||||
codegen-units = 1
|
codegen-units = 1
|
||||||
panic = 'abort'
|
panic = 'abort'
|
||||||
incremental = false
|
strip = true
|
||||||
opt-level = "s"
|
|
||||||
|
|
|
||||||
57
builder.sh
57
builder.sh
|
|
@ -6,24 +6,21 @@ LINUX_X86_TARGET="i686-unknown-linux-musl"
|
||||||
WIN_TARGET="x86_64-pc-windows-gnu"
|
WIN_TARGET="x86_64-pc-windows-gnu"
|
||||||
ARMV7_TARGET="armv7-unknown-linux-gnueabihf"
|
ARMV7_TARGET="armv7-unknown-linux-gnueabihf"
|
||||||
AARCH_TARGET="aarch64-unknown-linux-gnu"
|
AARCH_TARGET="aarch64-unknown-linux-gnu"
|
||||||
OSX_TARGET="x86_64-apple-darwin"
|
# OSX_TARGET="x86_64-apple-darwin"
|
||||||
MANPAGE_DIR="./$NAME.1"
|
MANPAGE_DIR="./$NAME.1"
|
||||||
|
BIN_OUTPUT_DIR="./ghbinaries"
|
||||||
|
|
||||||
if ! systemctl is-active docker >/dev/null 2>&1; then
|
rm -rf "$BIN_OUTPUT_DIR"
|
||||||
echo "Docker is not running. Starting docker."
|
mkdir -p "$BIN_OUTPUT_DIR"
|
||||||
if ! sudo systemctl start docker; then
|
|
||||||
echo "Failed to start docker."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Linux build
|
# Linux build
|
||||||
echo "Building Linux artifact."
|
echo "Building Linux artifact."
|
||||||
if cargo build -q --release --target="$LINUX_TARGET"; then
|
if cross build -q --release --target="$LINUX_TARGET"; then
|
||||||
echo "Linux artifact build: SUCCESS"
|
echo "Linux artifact build: SUCCESS"
|
||||||
cp "target/$LINUX_TARGET/release/$NAME" "target/$LINUX_TARGET/release/$NAME-linux"
|
cp "target/$LINUX_TARGET/release/$NAME" "target/$LINUX_TARGET/release/$NAME-linux"
|
||||||
strip "target/$LINUX_TARGET/release/$NAME-linux"
|
strip "target/$LINUX_TARGET/release/$NAME-linux"
|
||||||
sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"target/$LINUX_TARGET/release/$NAME-linux.sha512"
|
sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"$BIN_OUTPUT_DIR/$NAME-linux.sha512"
|
||||||
|
zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-x64.zip" "target/$LINUX_TARGET/release/$NAME-linux"
|
||||||
else
|
else
|
||||||
echo "Linux artifact build: FAILED"
|
echo "Linux artifact build: FAILED"
|
||||||
fi
|
fi
|
||||||
|
|
@ -34,7 +31,8 @@ if cross build -q --release --target="$LINUX_X86_TARGET"; then
|
||||||
echo "Linux x86 artifact build: SUCCESS"
|
echo "Linux x86 artifact build: SUCCESS"
|
||||||
cp "target/$LINUX_X86_TARGET/release/$NAME" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
|
cp "target/$LINUX_X86_TARGET/release/$NAME" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
|
||||||
strip "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
|
strip "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
|
||||||
sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"target/$LINUX_X86_TARGET/release/$NAME-linux-i386.sha512"
|
sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"$BIN_OUTPUT_DIR/$NAME-linux-i386.sha512"
|
||||||
|
zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-i386.zip" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
|
||||||
else
|
else
|
||||||
echo "Linux x86 artifact build: FAILED"
|
echo "Linux x86 artifact build: FAILED"
|
||||||
fi
|
fi
|
||||||
|
|
@ -45,7 +43,8 @@ if cross build -q --release --target="$WIN_TARGET"; then
|
||||||
echo "Windows artifact build: SUCCESS"
|
echo "Windows artifact build: SUCCESS"
|
||||||
cp "target/$WIN_TARGET/release/$NAME.exe" "target/$WIN_TARGET/release/$NAME-windows.exe"
|
cp "target/$WIN_TARGET/release/$NAME.exe" "target/$WIN_TARGET/release/$NAME-windows.exe"
|
||||||
strip "target/$WIN_TARGET/release/$NAME-windows.exe"
|
strip "target/$WIN_TARGET/release/$NAME-windows.exe"
|
||||||
sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"target/$WIN_TARGET/release/$NAME-windows.exe.sha512"
|
sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"$BIN_OUTPUT_DIR/$NAME-windows.exe.sha512"
|
||||||
|
zip -q -j "$BIN_OUTPUT_DIR/$NAME-windows.zip" "target/$WIN_TARGET/release/$NAME-windows.exe"
|
||||||
else
|
else
|
||||||
echo "Windows artifact build: FAILED"
|
echo "Windows artifact build: FAILED"
|
||||||
fi
|
fi
|
||||||
|
|
@ -56,7 +55,8 @@ if cross build -q --release --target="$ARMV7_TARGET"; then
|
||||||
echo "ARMv7 artifact build: SUCCESS"
|
echo "ARMv7 artifact build: SUCCESS"
|
||||||
cp "target/$ARMV7_TARGET/release/$NAME" "target/$ARMV7_TARGET/release/$NAME-armv7"
|
cp "target/$ARMV7_TARGET/release/$NAME" "target/$ARMV7_TARGET/release/$NAME-armv7"
|
||||||
strip "target/$ARMV7_TARGET/release/$NAME-armv7"
|
strip "target/$ARMV7_TARGET/release/$NAME-armv7"
|
||||||
sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"target/$ARMV7_TARGET/release/$NAME-armv7.sha512"
|
sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"$BIN_OUTPUT_DIR/$NAME-armv7.sha512"
|
||||||
|
zip -q -j "$BIN_OUTPUT_DIR/$NAME-armv7.zip" "target/$ARMV7_TARGET/release/$NAME-armv7"
|
||||||
else
|
else
|
||||||
echo "ARMv7 artifact build: FAILED"
|
echo "ARMv7 artifact build: FAILED"
|
||||||
fi
|
fi
|
||||||
|
|
@ -67,21 +67,23 @@ if cross build -q --release --target="$AARCH_TARGET"; then
|
||||||
echo "Aarch64 artifact build: SUCCESS"
|
echo "Aarch64 artifact build: SUCCESS"
|
||||||
cp "target/$AARCH_TARGET/release/$NAME" "target/$AARCH_TARGET/release/$NAME-aarch64"
|
cp "target/$AARCH_TARGET/release/$NAME" "target/$AARCH_TARGET/release/$NAME-aarch64"
|
||||||
strip "target/$AARCH_TARGET/release/$NAME-aarch64"
|
strip "target/$AARCH_TARGET/release/$NAME-aarch64"
|
||||||
sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"target/$AARCH_TARGET/release/$NAME-aarch64.sha512"
|
sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"$BIN_OUTPUT_DIR/$NAME-aarch64.sha512"
|
||||||
|
zip -q -j "$BIN_OUTPUT_DIR/$NAME-aarch64.zip" "target/$AARCH_TARGET/release/$NAME-aarch64"
|
||||||
else
|
else
|
||||||
echo "Aarch64 artifact build: FAILED"
|
echo "Aarch64 artifact build: FAILED"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Mac OS build
|
# # Mac OS build
|
||||||
echo "Building OSX artifact."
|
# echo "Building OSX artifact."
|
||||||
if CC=o64-clang CXX=o64-clang++ LIBZ_SYS_STATIC=1 cargo build -q --release --target="$OSX_TARGET"; then
|
# if CC=o64-clang CXX=o64-clang++ LIBZ_SYS_STATIC=1 cargo build -q --release --target="$OSX_TARGET"; then
|
||||||
echo "OSX artifact build: SUCCESS"
|
# echo "OSX artifact build: SUCCESS"
|
||||||
cp "target/$OSX_TARGET/release/$NAME" "target/$OSX_TARGET/release/$NAME-osx"
|
# cp "target/$OSX_TARGET/release/$NAME" "target/$OSX_TARGET/release/$NAME-osx"
|
||||||
strip "target/$OSX_TARGET/release/$NAME-osx"
|
# strip "target/$OSX_TARGET/release/$NAME-osx"
|
||||||
sha512sum "target/$OSX_TARGET/release/$NAME-osx" >"target/$OSX_TARGET/release/$NAME-osx.sha512"
|
# sha512sum "target/$OSX_TARGET/release/$NAME-osx" >"target/$OSX_TARGET/release/$NAME-osx.sha512"
|
||||||
else
|
# zip -q "$BIN_OUTPUT_DIR/$NAME-osx.zip" "target/$OSX_TARGET/release/$NAME-osx"
|
||||||
echo "OSX artifact build: FAILED"
|
# else
|
||||||
fi
|
# echo "OSX artifact build: FAILED"
|
||||||
|
# fi
|
||||||
|
|
||||||
echo "Creating manpage..."
|
echo "Creating manpage..."
|
||||||
if command -v help2man >/dev/null; then
|
if command -v help2man >/dev/null; then
|
||||||
|
|
@ -93,10 +95,3 @@ if command -v help2man >/dev/null; then
|
||||||
else
|
else
|
||||||
echo "Please install the help2man package."
|
echo "Please install the help2man package."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Stop docker
|
|
||||||
echo "Stopping docker."
|
|
||||||
if ! sudo systemctl stop docker; then
|
|
||||||
echo "Failed to stop docker."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
|
||||||
81
fhc.1
81
fhc.1
|
|
@ -1,65 +1,58 @@
|
||||||
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.1.
|
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3.
|
||||||
.TH FHC "1" "April 2022" "FHC 0.7.1" "User Commands"
|
.TH FHC "1" "August 2025" "fhc 0.9.1" "User Commands"
|
||||||
.SH NAME
|
.SH NAME
|
||||||
FHC \- manual page for FHC 0.7.1
|
fhc \- manual page for fhc 0.9.1
|
||||||
|
.SH SYNOPSIS
|
||||||
|
.B fhc
|
||||||
|
[\fI\,OPTIONS\/\fR]
|
||||||
.SH DESCRIPTION
|
.SH DESCRIPTION
|
||||||
FHC 0.7.1
|
|
||||||
Eduard Tolosa <edu4rdshl@protonmail.com>
|
|
||||||
Fast HTTP Checker.
|
Fast HTTP Checker.
|
||||||
.SS "USAGE:"
|
.SH OPTIONS
|
||||||
.IP
|
|
||||||
fhc [FLAGS] [OPTIONS]
|
|
||||||
.SS "FLAGS:"
|
|
||||||
.TP
|
.TP
|
||||||
\fB\-1\fR, \fB\-\-1xx\fR
|
\fB\-t\fR, \fB\-\-threads\fR <THREADS>
|
||||||
Show URLs with 100\-199 response codes only.
|
Number of threads to use [default: 50]
|
||||||
.TP
|
.TP
|
||||||
\fB\-2\fR, \fB\-\-2xx\fR
|
\fB\-\-timeout\fR <TIMEOUT>
|
||||||
Show URLs with 200\-299 response codes only.
|
Timeout in seconds [default: 3]
|
||||||
.TP
|
.TP
|
||||||
\fB\-3\fR, \fB\-\-3xx\fR
|
\fB\-s\fR, \fB\-\-show\-full\-data\fR
|
||||||
Show URLs with 300\-399 response codes only.
|
Show HTTP status codes, final URL and domain
|
||||||
.TP
|
.TP
|
||||||
\fB\-4\fR, \fB\-\-4xx\fR
|
\fB\-d\fR, \fB\-\-domain\fR <DOMAIN>
|
||||||
Show URLs with 400\-499 response codes only.
|
Domain to check \- can be omitted if using stdin
|
||||||
.TP
|
.TP
|
||||||
\fB\-5\fR, \fB\-\-5xx\fR
|
\fB\-r\fR, \fB\-\-retries\fR <RETRIES>
|
||||||
Show URLs with 500\-599 response codes only.
|
Number of retries [default: 1]
|
||||||
|
.TP
|
||||||
|
\fB\-L\fR, \fB\-\-max\-redirects\fR <MAX_REDIRECTS>
|
||||||
|
Maximum number of redirects, disabled by default [default: 0]
|
||||||
|
.TP
|
||||||
|
\fB\-b\fR, \fB\-\-bruteforce\fR
|
||||||
|
Enable bruteforce mode
|
||||||
|
.TP
|
||||||
|
\fB\-f\fR, \fB\-\-filter\-codes\fR <FILTER_CODES>
|
||||||
|
Filter status codes. A comma separated list can be used
|
||||||
|
.TP
|
||||||
|
\fB\-e\fR, \fB\-\-exclude\-codes\fR <EXCLUDE_CODES>
|
||||||
|
Exclude status codes. A comma separated list can be used
|
||||||
|
.TP
|
||||||
|
\fB\-q\fR, \fB\-\-quiet\fR
|
||||||
|
Quiet mode. This will suppress all fancy output except for the final results
|
||||||
.TP
|
.TP
|
||||||
\fB\-h\fR, \fB\-\-help\fR
|
\fB\-h\fR, \fB\-\-help\fR
|
||||||
Prints help information
|
Print help
|
||||||
.TP
|
|
||||||
\fB\-s\fR, \fB\-\-show\-codes\fR
|
|
||||||
Show status codes for discovered hosts.
|
|
||||||
.TP
|
.TP
|
||||||
\fB\-V\fR, \fB\-\-version\fR
|
\fB\-V\fR, \fB\-\-version\fR
|
||||||
Prints version information
|
Print version
|
||||||
.SS "OPTIONS:"
|
|
||||||
.TP
|
|
||||||
\fB\-d\fR, \fB\-\-domain\fR <domain>
|
|
||||||
Target domain. When it's specified, a wordlist can be used from stdin for
|
|
||||||
bruteforcing.
|
|
||||||
.TP
|
|
||||||
\fB\-L\fR, \fB\-\-max\-redirects\fR <max\-redirects>
|
|
||||||
Max number of redirects. Default: 0
|
|
||||||
.TP
|
|
||||||
\fB\-r\fR, \fB\-\-retries\fR <retries>
|
|
||||||
Max number of http probes per target.
|
|
||||||
.TP
|
|
||||||
\fB\-t\fR, \fB\-\-threads\fR <threads>
|
|
||||||
Number of threads. Default: 50
|
|
||||||
.TP
|
|
||||||
\fB\-\-timeout\fR <timeout>
|
|
||||||
Timeout in seconds. Default: 3
|
|
||||||
.SH "SEE ALSO"
|
.SH "SEE ALSO"
|
||||||
The full documentation for
|
The full documentation for
|
||||||
.B FHC
|
.B fhc
|
||||||
is maintained as a Texinfo manual. If the
|
is maintained as a Texinfo manual. If the
|
||||||
.B info
|
.B info
|
||||||
and
|
and
|
||||||
.B FHC
|
.B fhc
|
||||||
programs are properly installed at your site, the command
|
programs are properly installed at your site, the command
|
||||||
.IP
|
.IP
|
||||||
.B info FHC
|
.B info fhc
|
||||||
.PP
|
.PP
|
||||||
should give you access to the complete manual.
|
should give you access to the complete manual.
|
||||||
|
|
|
||||||
|
|
@ -10,16 +10,16 @@ pub struct Cli {
|
||||||
/// Timeout in seconds
|
/// Timeout in seconds
|
||||||
pub timeout: u64,
|
pub timeout: u64,
|
||||||
#[clap(short, long)]
|
#[clap(short, long)]
|
||||||
/// Show HTTP status codes
|
/// Show HTTP status codes, final URL and domain
|
||||||
pub show_codes: bool,
|
pub show_full_data: bool,
|
||||||
#[clap(short, long)]
|
#[clap(short, long)]
|
||||||
/// Domain to check - can be omitted if using stdin
|
/// Domain to check - can be omitted if using stdin
|
||||||
pub domain: Option<String>,
|
pub domain: Option<String>,
|
||||||
#[clap(short, long, default_value = "1")]
|
#[clap(short, long, default_value = "1")]
|
||||||
/// Number of retries
|
/// Number of retries
|
||||||
pub retries: usize,
|
pub retries: usize,
|
||||||
#[clap(short = 'L', long, default_value = "10")]
|
#[clap(short = 'L', long, default_value = "0")]
|
||||||
/// Maximum number of redirects
|
/// Maximum number of redirects, disabled by default.
|
||||||
pub max_redirects: usize,
|
pub max_redirects: usize,
|
||||||
#[clap(short, long)]
|
#[clap(short, long)]
|
||||||
/// Enable bruteforce mode
|
/// Enable bruteforce mode
|
||||||
|
|
|
||||||
153
src/httplib.rs
153
src/httplib.rs
|
|
@ -4,7 +4,7 @@ use {
|
||||||
utils,
|
utils,
|
||||||
},
|
},
|
||||||
futures::stream::StreamExt,
|
futures::stream::StreamExt,
|
||||||
rand::{distributions::Alphanumeric, thread_rng as rng, Rng},
|
rand::{distr::Alphanumeric, rng, Rng},
|
||||||
reqwest::{
|
reqwest::{
|
||||||
header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT},
|
header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT},
|
||||||
redirect::Policy,
|
redirect::Policy,
|
||||||
|
|
@ -15,34 +15,37 @@ use {
|
||||||
};
|
};
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub async fn return_http_data(options: &LibOptions) -> HashMap<String, HttpData> {
|
#[allow(clippy::similar_names)]
|
||||||
|
pub async fn return_http_data(options: &LibOptions, from_cli: bool) -> HashMap<String, HttpData> {
|
||||||
let threads = options.hosts.len().min(options.threads);
|
let threads = options.hosts.len().min(options.threads);
|
||||||
|
|
||||||
let filter_codes = options.filter_codes.as_deref().unwrap_or_default();
|
let filter_codes = options.filter_codes.as_deref().unwrap_or_default();
|
||||||
let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default();
|
let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default();
|
||||||
|
|
||||||
futures::stream::iter(options.hosts.clone().into_iter().map(|host| {
|
futures::stream::iter(options.hosts.iter().map(|host| {
|
||||||
let user_agent = utils::return_random_user_agent(&options.user_agents);
|
let user_agent = utils::return_random_user_agent(&options.user_agents);
|
||||||
|
|
||||||
async move {
|
async move {
|
||||||
let mut http_data = HttpData::default();
|
let mut http_data = HttpData::new(host.clone());
|
||||||
let mut response = None;
|
let mut response = None;
|
||||||
|
|
||||||
// Attempt both HTTPS and HTTP requests, retry if necessary
|
|
||||||
for _ in 0..options.retries {
|
for _ in 0..options.retries {
|
||||||
let https_req = options
|
let https_req = options
|
||||||
.client
|
.client
|
||||||
.get(format!("https://{}", host))
|
.get(format!("https://{host}"))
|
||||||
.header(USER_AGENT, &user_agent)
|
.header(USER_AGENT, &user_agent)
|
||||||
.send();
|
.send();
|
||||||
|
|
||||||
let http_req = options
|
let http_req = options
|
||||||
.client
|
.client
|
||||||
.get(format!("http://{}", host))
|
.get(format!("http://{host}"))
|
||||||
.header(USER_AGENT, &user_agent)
|
.header(USER_AGENT, &user_agent)
|
||||||
.send();
|
.send();
|
||||||
|
|
||||||
response = https_req.await.or(http_req.await).ok();
|
response = tokio::select! {
|
||||||
|
https_result = https_req => https_result.ok(),
|
||||||
|
http_result = http_req => http_result.ok(),
|
||||||
|
};
|
||||||
|
|
||||||
if response.is_some() {
|
if response.is_some() {
|
||||||
break;
|
break;
|
||||||
|
|
@ -50,36 +53,45 @@ pub async fn return_http_data(options: &LibOptions) -> HashMap<String, HttpData>
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(resp) = response {
|
if let Some(resp) = response {
|
||||||
http_data.checked_host = host.clone();
|
// Those are always set
|
||||||
|
http_data.protocol = resp.url().scheme().to_string();
|
||||||
|
http_data.status_code = resp.status().as_u16();
|
||||||
http_data.final_url = resp.url().to_string();
|
http_data.final_url = resp.url().to_string();
|
||||||
|
|
||||||
if options.assign_response_data {
|
if !from_cli {
|
||||||
http_data = assign_response_data(http_data, resp, options.return_filters).await;
|
assign_response_data(&mut http_data, resp, options.return_filters).await;
|
||||||
} else {
|
|
||||||
http_data.status_code = resp.status().as_u16();
|
|
||||||
http_data.http_status = "ACTIVE".to_string();
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
http_data.http_status = "INACTIVE".to_string();
|
http_data.http_status = "INACTIVE".to_string();
|
||||||
}
|
}
|
||||||
|
|
||||||
if !options.quiet_flag
|
if !options.quiet_flag
|
||||||
&& !http_data.checked_host.is_empty()
|
&& !http_data.final_url.is_empty()
|
||||||
&& (filter_codes.is_empty()
|
&& (filter_codes.is_empty()
|
||||||
|| filter_codes.contains(&http_data.status_code.to_string()))
|
|| filter_codes.contains(&http_data.status_code.to_string()))
|
||||||
&& (exclude_codes.is_empty()
|
&& (exclude_codes.is_empty()
|
||||||
|| !exclude_codes.contains(&http_data.status_code.to_string()))
|
|| !exclude_codes.contains(&http_data.status_code.to_string()))
|
||||||
{
|
{
|
||||||
if options.show_status_codes {
|
// Use faster I/O for high-throughput scenarios
|
||||||
println!(
|
use std::io::{self, Write};
|
||||||
|
let stdout = io::stdout();
|
||||||
|
let mut handle = stdout.lock();
|
||||||
|
|
||||||
|
if options.show_full_data {
|
||||||
|
let _ = writeln!(
|
||||||
|
handle,
|
||||||
"{},[{}],[{}]",
|
"{},[{}],[{}]",
|
||||||
http_data.checked_host, http_data.final_url, http_data.status_code
|
http_data.checked_host, http_data.final_url, http_data.status_code
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
println!("{},[{}]", http_data.checked_host, http_data.final_url);
|
let _ = writeln!(
|
||||||
|
handle,
|
||||||
|
"{}://{}",
|
||||||
|
http_data.protocol, http_data.checked_host
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
(host, http_data)
|
(host.clone(), http_data)
|
||||||
}
|
}
|
||||||
}))
|
}))
|
||||||
.buffer_unordered(threads)
|
.buffer_unordered(threads)
|
||||||
|
|
@ -100,30 +112,33 @@ pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client {
|
||||||
.redirect(policy)
|
.redirect(policy)
|
||||||
.danger_accept_invalid_certs(true)
|
.danger_accept_invalid_certs(true)
|
||||||
.use_native_tls()
|
.use_native_tls()
|
||||||
|
.pool_max_idle_per_host(50)
|
||||||
|
.pool_idle_timeout(std::time::Duration::from_secs(30))
|
||||||
|
.tcp_keepalive(std::time::Duration::from_secs(60))
|
||||||
.build()
|
.build()
|
||||||
.expect("Failed to create HTTP client")
|
.expect("Failed to create HTTP client")
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::field_reassign_with_default)]
|
#[allow(clippy::field_reassign_with_default)]
|
||||||
pub async fn assign_response_data(
|
pub async fn assign_response_data(http_data: &mut HttpData, resp: Response, return_filters: bool) {
|
||||||
mut http_data: HttpData,
|
|
||||||
resp: Response,
|
|
||||||
return_filters: bool,
|
|
||||||
) -> HttpData {
|
|
||||||
let headers = resp.headers().clone();
|
let headers = resp.headers().clone();
|
||||||
let url = resp.url().clone();
|
let url = resp.url().clone();
|
||||||
|
|
||||||
http_data.http_status = "ACTIVE".to_string();
|
http_data.http_status = "ACTIVE".to_string();
|
||||||
http_data.status_code = resp.status().as_u16();
|
|
||||||
http_data.final_url = url.to_string();
|
|
||||||
http_data.protocol = url.scheme().to_string();
|
|
||||||
http_data.content_type = headers
|
http_data.content_type = headers
|
||||||
.get(CONTENT_TYPE)
|
.get(CONTENT_TYPE)
|
||||||
.and_then(|v| v.to_str().ok())
|
.and_then(|v| v.to_str().ok())
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
let full_body = (resp.text().await).unwrap_or_default();
|
let full_body = {
|
||||||
|
const MAX_BODY_SIZE: usize = 1024 * 1024; // 1MB limit
|
||||||
|
match resp.text().await {
|
||||||
|
Ok(text) if text.len() <= MAX_BODY_SIZE => text,
|
||||||
|
Ok(text) => text.chars().take(MAX_BODY_SIZE).collect(),
|
||||||
|
Err(_) => String::new(),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
http_data.content_length = headers
|
http_data.content_length = headers
|
||||||
.get(CONTENT_LENGTH)
|
.get(CONTENT_LENGTH)
|
||||||
|
|
@ -132,10 +147,14 @@ pub async fn assign_response_data(
|
||||||
|
|
||||||
http_data.headers = format!("{headers:?}");
|
http_data.headers = format!("{headers:?}");
|
||||||
|
|
||||||
return_title_and_body(&mut http_data, &full_body);
|
return_title_and_body(http_data, &full_body);
|
||||||
|
|
||||||
http_data.words_count = full_body.split_whitespace().count();
|
let lines_count = full_body.lines().count();
|
||||||
http_data.lines = full_body.lines().count() + 1;
|
let words_count = full_body.split_whitespace().count();
|
||||||
|
|
||||||
|
http_data.words_count = words_count;
|
||||||
|
http_data.lines = lines_count + 1;
|
||||||
|
http_data.points_to_another_host = url.host_str() != Some(&http_data.checked_host);
|
||||||
|
|
||||||
if return_filters {
|
if return_filters {
|
||||||
let host = url.host_str().unwrap_or_default();
|
let host = url.host_str().unwrap_or_default();
|
||||||
|
|
@ -143,42 +162,29 @@ pub async fn assign_response_data(
|
||||||
let user_agents = utils::user_agents();
|
let user_agents = utils::user_agents();
|
||||||
http_data.bad_data = return_filters_data(host, client, user_agents).await;
|
http_data.bad_data = return_filters_data(host, client, user_agents).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
http_data
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn return_title_and_body(http_data: &mut HttpData, body: &str) {
|
pub fn return_title_and_body(http_data: &mut HttpData, body: &str) {
|
||||||
let document = Html::parse_document(body);
|
let document = Html::parse_document(body);
|
||||||
|
|
||||||
// Return title
|
let title_selector = Selector::parse("title").ok();
|
||||||
match Selector::parse("title") {
|
let body_selector = Selector::parse("body").ok();
|
||||||
Ok(selector) => {
|
|
||||||
if let Some(title_element) = document.select(&selector).next() {
|
if let Some(title_sel) = &title_selector {
|
||||||
http_data.title = title_element.inner_html();
|
http_data.title = document
|
||||||
} else {
|
.select(title_sel)
|
||||||
http_data.title = "NULL".to_string();
|
.next()
|
||||||
}
|
.map(|element| element.inner_html())
|
||||||
}
|
.unwrap_or_else(|| "NULL".to_string());
|
||||||
Err(err) => {
|
|
||||||
eprintln!("Failed to parse selector: {err:?}");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return body
|
if let Some(body_sel) = &body_selector {
|
||||||
match Selector::parse("body") {
|
http_data.body = document
|
||||||
Ok(selector) => {
|
.select(body_sel)
|
||||||
if let Some(body_element) = document.select(&selector).next() {
|
.next()
|
||||||
http_data.body = body_element.inner_html();
|
.map(|element| element.inner_html())
|
||||||
} else {
|
.unwrap_or_else(|| "NULL".to_string());
|
||||||
http_data.body = "NULL".to_string();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
eprintln!("Failed to parse selector: {err:?}");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
drop(document);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn return_filters_data(
|
pub async fn return_filters_data(
|
||||||
|
|
@ -211,28 +217,23 @@ pub async fn return_filters_data(
|
||||||
user_agents: user_agents_list,
|
user_agents: user_agents_list,
|
||||||
retries: 1,
|
retries: 1,
|
||||||
threads,
|
threads,
|
||||||
assign_response_data: true,
|
|
||||||
quiet_flag: true,
|
quiet_flag: true,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
let data = return_http_data(&lib_options).await;
|
let data = return_http_data(&lib_options, false).await;
|
||||||
|
|
||||||
data.values()
|
data.values().for_each(|http_data| {
|
||||||
.map(|http_data| {
|
http_filters
|
||||||
http_filters
|
.bad_http_lengths
|
||||||
.bad_http_lengths
|
.push(http_data.content_length.to_string());
|
||||||
.append(&mut vec![http_data.content_length.to_string()]);
|
http_filters
|
||||||
http_filters.bad_words_numbers.append(&mut vec![http_data
|
.bad_words_numbers
|
||||||
.body
|
.push(http_data.words_count.to_string());
|
||||||
.split(' ')
|
http_filters
|
||||||
.count()
|
.bad_lines_numbers
|
||||||
.to_string()]);
|
.push(http_data.lines.to_string());
|
||||||
http_filters
|
});
|
||||||
.bad_lines_numbers
|
|
||||||
.append(&mut vec![http_data.lines.to_string()]);
|
|
||||||
})
|
|
||||||
.for_each(drop);
|
|
||||||
|
|
||||||
http_filters
|
http_filters
|
||||||
}
|
}
|
||||||
|
|
|
||||||
38
src/main.rs
38
src/main.rs
|
|
@ -10,19 +10,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
// Eval args
|
// Eval args
|
||||||
let args = args::Cli::parse();
|
let args = args::Cli::parse();
|
||||||
|
|
||||||
let filter_codes = args.filter_codes;
|
|
||||||
let exclude_codes = args.exclude_codes;
|
|
||||||
let threads = args.threads;
|
|
||||||
let retries = args.retries;
|
|
||||||
let timeout = args.timeout;
|
|
||||||
let max_redirects = args.max_redirects;
|
|
||||||
let user_agents_list = utils::user_agents();
|
|
||||||
let show_status_codes = args.show_codes;
|
|
||||||
|
|
||||||
let client = httplib::return_http_client(timeout, max_redirects);
|
|
||||||
|
|
||||||
let mut buffer = String::new();
|
let mut buffer = String::new();
|
||||||
|
|
||||||
let mut hosts = HashSet::new();
|
let mut hosts = HashSet::new();
|
||||||
|
|
||||||
if args.domain.is_some() {
|
if args.domain.is_some() {
|
||||||
|
|
@ -39,29 +27,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
} else {
|
} else {
|
||||||
io::stdin().read_to_string(&mut buffer).await?;
|
io::stdin().read_to_string(&mut buffer).await?;
|
||||||
hosts = buffer.lines().map(str::to_owned).collect();
|
hosts = buffer.lines().map(str::to_owned).collect();
|
||||||
};
|
}
|
||||||
|
|
||||||
let lib_options = LibOptions {
|
let lib_options = LibOptions {
|
||||||
hosts,
|
hosts,
|
||||||
client,
|
client: httplib::return_http_client(args.timeout, args.max_redirects),
|
||||||
user_agents: user_agents_list,
|
user_agents: utils::user_agents(),
|
||||||
retries,
|
retries: args.retries,
|
||||||
threads,
|
threads: args.threads,
|
||||||
filter_codes,
|
filter_codes: args.filter_codes,
|
||||||
exclude_codes,
|
exclude_codes: args.exclude_codes,
|
||||||
show_status_codes,
|
show_full_data: args.show_full_data,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
if !args.quiet {
|
if !args.quiet && args.show_full_data {
|
||||||
if show_status_codes {
|
println!("DOMAIN,[FINAL_URL],[STATUS_CODE]");
|
||||||
println!("DOMAIN,[FINAL_URL],[STATUS_CODE]");
|
|
||||||
} else {
|
|
||||||
println!("DOMAIN,[FINAL_URL]");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let _ = httplib::return_http_data(&lib_options).await;
|
let _ = httplib::return_http_data(&lib_options, true).await;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,17 @@ pub struct HttpData {
|
||||||
pub bad_data: HTTPFilters,
|
pub bad_data: HTTPFilters,
|
||||||
pub html_file_path: String,
|
pub html_file_path: String,
|
||||||
pub screenshot_data: Vec<u8>,
|
pub screenshot_data: Vec<u8>,
|
||||||
|
pub points_to_another_host: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl HttpData {
|
||||||
|
#[inline]
|
||||||
|
pub fn new(host: String) -> Self {
|
||||||
|
Self {
|
||||||
|
checked_host: host,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default)]
|
#[derive(Clone, Debug, Default)]
|
||||||
|
|
@ -37,7 +48,6 @@ pub struct LibOptions {
|
||||||
pub return_filters: bool,
|
pub return_filters: bool,
|
||||||
pub filter_codes: Option<String>,
|
pub filter_codes: Option<String>,
|
||||||
pub exclude_codes: Option<String>,
|
pub exclude_codes: Option<String>,
|
||||||
pub show_status_codes: bool,
|
pub show_full_data: bool,
|
||||||
pub assign_response_data: bool,
|
|
||||||
pub quiet_flag: bool,
|
pub quiet_flag: bool,
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
use rand::{seq::SliceRandom, thread_rng};
|
use rand::{rng, seq::IndexedRandom};
|
||||||
|
|
||||||
#[must_use]
|
#[must_use]
|
||||||
pub fn user_agents() -> Vec<String> {
|
pub fn user_agents() -> Vec<String> {
|
||||||
|
|
@ -37,7 +37,7 @@ pub fn user_agents() -> Vec<String> {
|
||||||
pub fn return_random_user_agent(strings: &[String]) -> String {
|
pub fn return_random_user_agent(strings: &[String]) -> String {
|
||||||
let empty_string = String::new();
|
let empty_string = String::new();
|
||||||
strings
|
strings
|
||||||
.choose(&mut thread_rng())
|
.choose(&mut rng())
|
||||||
.unwrap_or(&empty_string)
|
.unwrap_or(&empty_string)
|
||||||
.to_string()
|
.to_string()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue