mirror of
https://github.com/edu4rdshl/fhc.git
synced 2026-07-17 23:24:50 +00:00
Compare commits
20 commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ed011be8a6 | |||
| 70ee3429ae | |||
| 7b39d6ebc8 | |||
| e7917b1e47 | |||
| 06b99f9b5d | |||
| 12b42d9a05 | |||
| 6701590efd | |||
| cc37ccbcac | |||
| fa58e6aa7e | |||
| 291f8dfe9e | |||
| 99db1a4b29 | |||
| c0ba951955 | |||
| 002a612e70 | |||
| cdd2d778b4 | |||
| 973c6fc1c9 | |||
| 9d55a00335 | |||
| 200eac74dd | |||
| ad731731b7 | |||
| ede2c96e0f | |||
| 3a0674997e |
10 changed files with 1034 additions and 676 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -2,3 +2,4 @@
|
|||
/hosts.txt
|
||||
*.txt
|
||||
armbuilder.sh
|
||||
/ghbinaries
|
||||
|
|
|
|||
1335
Cargo.lock
generated
1335
Cargo.lock
generated
File diff suppressed because it is too large
Load diff
19
Cargo.toml
19
Cargo.toml
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "fhc"
|
||||
version = "0.8.0"
|
||||
version = "0.9.1"
|
||||
authors = ["Eduard Tolosa <edu4rdshl@protonmail.com>"]
|
||||
edition = "2018"
|
||||
description = "Fast HTTP Checker."
|
||||
|
|
@ -12,22 +12,21 @@ readme = "README.md"
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
tokio = { version = "1.40.0", features = ["full", "io-util"] }
|
||||
tokio = { version = "1.47.0", features = ["full", "io-util"] }
|
||||
futures = "0.3.31"
|
||||
clap = { version = "4.5.20", features = ["derive"] }
|
||||
reqwest = { version = "0.12.8", features = [
|
||||
clap = { version = "4.5.41", features = ["derive"] }
|
||||
reqwest = { version = "0.12.22", features = [
|
||||
"hickory-dns",
|
||||
"rustls-tls",
|
||||
"native-tls",
|
||||
] }
|
||||
openssl = { version = "0.10.66", features = ["vendored"] }
|
||||
rand = "0.8.5"
|
||||
scraper = "0.20.0"
|
||||
openssl = { version = "0.10.73", features = ["vendored"] }
|
||||
rand = "0.9.2"
|
||||
scraper = "0.23.1"
|
||||
async-recursion = "1.1.1"
|
||||
|
||||
[profile.release]
|
||||
lto = 'thin'
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
panic = 'abort'
|
||||
incremental = false
|
||||
opt-level = "s"
|
||||
strip = true
|
||||
|
|
|
|||
57
builder.sh
57
builder.sh
|
|
@ -6,24 +6,21 @@ LINUX_X86_TARGET="i686-unknown-linux-musl"
|
|||
WIN_TARGET="x86_64-pc-windows-gnu"
|
||||
ARMV7_TARGET="armv7-unknown-linux-gnueabihf"
|
||||
AARCH_TARGET="aarch64-unknown-linux-gnu"
|
||||
OSX_TARGET="x86_64-apple-darwin"
|
||||
# OSX_TARGET="x86_64-apple-darwin"
|
||||
MANPAGE_DIR="./$NAME.1"
|
||||
BIN_OUTPUT_DIR="./ghbinaries"
|
||||
|
||||
if ! systemctl is-active docker >/dev/null 2>&1; then
|
||||
echo "Docker is not running. Starting docker."
|
||||
if ! sudo systemctl start docker; then
|
||||
echo "Failed to start docker."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
rm -rf "$BIN_OUTPUT_DIR"
|
||||
mkdir -p "$BIN_OUTPUT_DIR"
|
||||
|
||||
# Linux build
|
||||
echo "Building Linux artifact."
|
||||
if cargo build -q --release --target="$LINUX_TARGET"; then
|
||||
if cross build -q --release --target="$LINUX_TARGET"; then
|
||||
echo "Linux artifact build: SUCCESS"
|
||||
cp "target/$LINUX_TARGET/release/$NAME" "target/$LINUX_TARGET/release/$NAME-linux"
|
||||
strip "target/$LINUX_TARGET/release/$NAME-linux"
|
||||
sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"target/$LINUX_TARGET/release/$NAME-linux.sha512"
|
||||
sha512sum "target/$LINUX_TARGET/release/$NAME-linux" >"$BIN_OUTPUT_DIR/$NAME-linux.sha512"
|
||||
zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-x64.zip" "target/$LINUX_TARGET/release/$NAME-linux"
|
||||
else
|
||||
echo "Linux artifact build: FAILED"
|
||||
fi
|
||||
|
|
@ -34,7 +31,8 @@ if cross build -q --release --target="$LINUX_X86_TARGET"; then
|
|||
echo "Linux x86 artifact build: SUCCESS"
|
||||
cp "target/$LINUX_X86_TARGET/release/$NAME" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
|
||||
strip "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
|
||||
sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"target/$LINUX_X86_TARGET/release/$NAME-linux-i386.sha512"
|
||||
sha512sum "target/$LINUX_X86_TARGET/release/$NAME-linux-i386" >"$BIN_OUTPUT_DIR/$NAME-linux-i386.sha512"
|
||||
zip -q -j "$BIN_OUTPUT_DIR/$NAME-linux-i386.zip" "target/$LINUX_X86_TARGET/release/$NAME-linux-i386"
|
||||
else
|
||||
echo "Linux x86 artifact build: FAILED"
|
||||
fi
|
||||
|
|
@ -45,7 +43,8 @@ if cross build -q --release --target="$WIN_TARGET"; then
|
|||
echo "Windows artifact build: SUCCESS"
|
||||
cp "target/$WIN_TARGET/release/$NAME.exe" "target/$WIN_TARGET/release/$NAME-windows.exe"
|
||||
strip "target/$WIN_TARGET/release/$NAME-windows.exe"
|
||||
sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"target/$WIN_TARGET/release/$NAME-windows.exe.sha512"
|
||||
sha512sum "target/$WIN_TARGET/release/$NAME-windows.exe" >"$BIN_OUTPUT_DIR/$NAME-windows.exe.sha512"
|
||||
zip -q -j "$BIN_OUTPUT_DIR/$NAME-windows.zip" "target/$WIN_TARGET/release/$NAME-windows.exe"
|
||||
else
|
||||
echo "Windows artifact build: FAILED"
|
||||
fi
|
||||
|
|
@ -56,7 +55,8 @@ if cross build -q --release --target="$ARMV7_TARGET"; then
|
|||
echo "ARMv7 artifact build: SUCCESS"
|
||||
cp "target/$ARMV7_TARGET/release/$NAME" "target/$ARMV7_TARGET/release/$NAME-armv7"
|
||||
strip "target/$ARMV7_TARGET/release/$NAME-armv7"
|
||||
sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"target/$ARMV7_TARGET/release/$NAME-armv7.sha512"
|
||||
sha512sum "target/$ARMV7_TARGET/release/$NAME-armv7" >"$BIN_OUTPUT_DIR/$NAME-armv7.sha512"
|
||||
zip -q -j "$BIN_OUTPUT_DIR/$NAME-armv7.zip" "target/$ARMV7_TARGET/release/$NAME-armv7"
|
||||
else
|
||||
echo "ARMv7 artifact build: FAILED"
|
||||
fi
|
||||
|
|
@ -67,21 +67,23 @@ if cross build -q --release --target="$AARCH_TARGET"; then
|
|||
echo "Aarch64 artifact build: SUCCESS"
|
||||
cp "target/$AARCH_TARGET/release/$NAME" "target/$AARCH_TARGET/release/$NAME-aarch64"
|
||||
strip "target/$AARCH_TARGET/release/$NAME-aarch64"
|
||||
sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"target/$AARCH_TARGET/release/$NAME-aarch64.sha512"
|
||||
sha512sum "target/$AARCH_TARGET/release/$NAME-aarch64" >"$BIN_OUTPUT_DIR/$NAME-aarch64.sha512"
|
||||
zip -q -j "$BIN_OUTPUT_DIR/$NAME-aarch64.zip" "target/$AARCH_TARGET/release/$NAME-aarch64"
|
||||
else
|
||||
echo "Aarch64 artifact build: FAILED"
|
||||
fi
|
||||
|
||||
# Mac OS build
|
||||
echo "Building OSX artifact."
|
||||
if CC=o64-clang CXX=o64-clang++ LIBZ_SYS_STATIC=1 cargo build -q --release --target="$OSX_TARGET"; then
|
||||
echo "OSX artifact build: SUCCESS"
|
||||
cp "target/$OSX_TARGET/release/$NAME" "target/$OSX_TARGET/release/$NAME-osx"
|
||||
strip "target/$OSX_TARGET/release/$NAME-osx"
|
||||
sha512sum "target/$OSX_TARGET/release/$NAME-osx" >"target/$OSX_TARGET/release/$NAME-osx.sha512"
|
||||
else
|
||||
echo "OSX artifact build: FAILED"
|
||||
fi
|
||||
# # Mac OS build
|
||||
# echo "Building OSX artifact."
|
||||
# if CC=o64-clang CXX=o64-clang++ LIBZ_SYS_STATIC=1 cargo build -q --release --target="$OSX_TARGET"; then
|
||||
# echo "OSX artifact build: SUCCESS"
|
||||
# cp "target/$OSX_TARGET/release/$NAME" "target/$OSX_TARGET/release/$NAME-osx"
|
||||
# strip "target/$OSX_TARGET/release/$NAME-osx"
|
||||
# sha512sum "target/$OSX_TARGET/release/$NAME-osx" >"target/$OSX_TARGET/release/$NAME-osx.sha512"
|
||||
# zip -q "$BIN_OUTPUT_DIR/$NAME-osx.zip" "target/$OSX_TARGET/release/$NAME-osx"
|
||||
# else
|
||||
# echo "OSX artifact build: FAILED"
|
||||
# fi
|
||||
|
||||
echo "Creating manpage..."
|
||||
if command -v help2man >/dev/null; then
|
||||
|
|
@ -93,10 +95,3 @@ if command -v help2man >/dev/null; then
|
|||
else
|
||||
echo "Please install the help2man package."
|
||||
fi
|
||||
|
||||
# Stop docker
|
||||
echo "Stopping docker."
|
||||
if ! sudo systemctl stop docker; then
|
||||
echo "Failed to stop docker."
|
||||
exit 1
|
||||
fi
|
||||
|
|
|
|||
81
fhc.1
81
fhc.1
|
|
@ -1,65 +1,58 @@
|
|||
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.1.
|
||||
.TH FHC "1" "April 2022" "FHC 0.7.1" "User Commands"
|
||||
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3.
|
||||
.TH FHC "1" "August 2025" "fhc 0.9.1" "User Commands"
|
||||
.SH NAME
|
||||
FHC \- manual page for FHC 0.7.1
|
||||
fhc \- manual page for fhc 0.9.1
|
||||
.SH SYNOPSIS
|
||||
.B fhc
|
||||
[\fI\,OPTIONS\/\fR]
|
||||
.SH DESCRIPTION
|
||||
FHC 0.7.1
|
||||
Eduard Tolosa <edu4rdshl@protonmail.com>
|
||||
Fast HTTP Checker.
|
||||
.SS "USAGE:"
|
||||
.IP
|
||||
fhc [FLAGS] [OPTIONS]
|
||||
.SS "FLAGS:"
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
\fB\-1\fR, \fB\-\-1xx\fR
|
||||
Show URLs with 100\-199 response codes only.
|
||||
\fB\-t\fR, \fB\-\-threads\fR <THREADS>
|
||||
Number of threads to use [default: 50]
|
||||
.TP
|
||||
\fB\-2\fR, \fB\-\-2xx\fR
|
||||
Show URLs with 200\-299 response codes only.
|
||||
\fB\-\-timeout\fR <TIMEOUT>
|
||||
Timeout in seconds [default: 3]
|
||||
.TP
|
||||
\fB\-3\fR, \fB\-\-3xx\fR
|
||||
Show URLs with 300\-399 response codes only.
|
||||
\fB\-s\fR, \fB\-\-show\-full\-data\fR
|
||||
Show HTTP status codes, final URL and domain
|
||||
.TP
|
||||
\fB\-4\fR, \fB\-\-4xx\fR
|
||||
Show URLs with 400\-499 response codes only.
|
||||
\fB\-d\fR, \fB\-\-domain\fR <DOMAIN>
|
||||
Domain to check \- can be omitted if using stdin
|
||||
.TP
|
||||
\fB\-5\fR, \fB\-\-5xx\fR
|
||||
Show URLs with 500\-599 response codes only.
|
||||
\fB\-r\fR, \fB\-\-retries\fR <RETRIES>
|
||||
Number of retries [default: 1]
|
||||
.TP
|
||||
\fB\-L\fR, \fB\-\-max\-redirects\fR <MAX_REDIRECTS>
|
||||
Maximum number of redirects, disabled by default [default: 0]
|
||||
.TP
|
||||
\fB\-b\fR, \fB\-\-bruteforce\fR
|
||||
Enable bruteforce mode
|
||||
.TP
|
||||
\fB\-f\fR, \fB\-\-filter\-codes\fR <FILTER_CODES>
|
||||
Filter status codes. A comma separated list can be used
|
||||
.TP
|
||||
\fB\-e\fR, \fB\-\-exclude\-codes\fR <EXCLUDE_CODES>
|
||||
Exclude status codes. A comma separated list can be used
|
||||
.TP
|
||||
\fB\-q\fR, \fB\-\-quiet\fR
|
||||
Quiet mode. This will suppress all fancy output except for the final results
|
||||
.TP
|
||||
\fB\-h\fR, \fB\-\-help\fR
|
||||
Prints help information
|
||||
.TP
|
||||
\fB\-s\fR, \fB\-\-show\-codes\fR
|
||||
Show status codes for discovered hosts.
|
||||
Print help
|
||||
.TP
|
||||
\fB\-V\fR, \fB\-\-version\fR
|
||||
Prints version information
|
||||
.SS "OPTIONS:"
|
||||
.TP
|
||||
\fB\-d\fR, \fB\-\-domain\fR <domain>
|
||||
Target domain. When it's specified, a wordlist can be used from stdin for
|
||||
bruteforcing.
|
||||
.TP
|
||||
\fB\-L\fR, \fB\-\-max\-redirects\fR <max\-redirects>
|
||||
Max number of redirects. Default: 0
|
||||
.TP
|
||||
\fB\-r\fR, \fB\-\-retries\fR <retries>
|
||||
Max number of http probes per target.
|
||||
.TP
|
||||
\fB\-t\fR, \fB\-\-threads\fR <threads>
|
||||
Number of threads. Default: 50
|
||||
.TP
|
||||
\fB\-\-timeout\fR <timeout>
|
||||
Timeout in seconds. Default: 3
|
||||
Print version
|
||||
.SH "SEE ALSO"
|
||||
The full documentation for
|
||||
.B FHC
|
||||
.B fhc
|
||||
is maintained as a Texinfo manual. If the
|
||||
.B info
|
||||
and
|
||||
.B FHC
|
||||
.B fhc
|
||||
programs are properly installed at your site, the command
|
||||
.IP
|
||||
.B info FHC
|
||||
.B info fhc
|
||||
.PP
|
||||
should give you access to the complete manual.
|
||||
|
|
|
|||
|
|
@ -10,16 +10,16 @@ pub struct Cli {
|
|||
/// Timeout in seconds
|
||||
pub timeout: u64,
|
||||
#[clap(short, long)]
|
||||
/// Show HTTP status codes
|
||||
pub show_codes: bool,
|
||||
/// Show HTTP status codes, final URL and domain
|
||||
pub show_full_data: bool,
|
||||
#[clap(short, long)]
|
||||
/// Domain to check - can be omitted if using stdin
|
||||
pub domain: Option<String>,
|
||||
#[clap(short, long, default_value = "1")]
|
||||
/// Number of retries
|
||||
pub retries: usize,
|
||||
#[clap(short = 'L', long, default_value = "10")]
|
||||
/// Maximum number of redirects
|
||||
#[clap(short = 'L', long, default_value = "0")]
|
||||
/// Maximum number of redirects, disabled by default.
|
||||
pub max_redirects: usize,
|
||||
#[clap(short, long)]
|
||||
/// Enable bruteforce mode
|
||||
|
|
|
|||
153
src/httplib.rs
153
src/httplib.rs
|
|
@ -4,7 +4,7 @@ use {
|
|||
utils,
|
||||
},
|
||||
futures::stream::StreamExt,
|
||||
rand::{distributions::Alphanumeric, thread_rng as rng, Rng},
|
||||
rand::{distr::Alphanumeric, rng, Rng},
|
||||
reqwest::{
|
||||
header::{CONTENT_LENGTH, CONTENT_TYPE, USER_AGENT},
|
||||
redirect::Policy,
|
||||
|
|
@ -15,34 +15,37 @@ use {
|
|||
};
|
||||
|
||||
#[must_use]
|
||||
pub async fn return_http_data(options: &LibOptions) -> HashMap<String, HttpData> {
|
||||
#[allow(clippy::similar_names)]
|
||||
pub async fn return_http_data(options: &LibOptions, from_cli: bool) -> HashMap<String, HttpData> {
|
||||
let threads = options.hosts.len().min(options.threads);
|
||||
|
||||
let filter_codes = options.filter_codes.as_deref().unwrap_or_default();
|
||||
let exclude_codes = options.exclude_codes.as_deref().unwrap_or_default();
|
||||
|
||||
futures::stream::iter(options.hosts.clone().into_iter().map(|host| {
|
||||
futures::stream::iter(options.hosts.iter().map(|host| {
|
||||
let user_agent = utils::return_random_user_agent(&options.user_agents);
|
||||
|
||||
async move {
|
||||
let mut http_data = HttpData::default();
|
||||
let mut http_data = HttpData::new(host.clone());
|
||||
let mut response = None;
|
||||
|
||||
// Attempt both HTTPS and HTTP requests, retry if necessary
|
||||
for _ in 0..options.retries {
|
||||
let https_req = options
|
||||
.client
|
||||
.get(format!("https://{}", host))
|
||||
.get(format!("https://{host}"))
|
||||
.header(USER_AGENT, &user_agent)
|
||||
.send();
|
||||
|
||||
let http_req = options
|
||||
.client
|
||||
.get(format!("http://{}", host))
|
||||
.get(format!("http://{host}"))
|
||||
.header(USER_AGENT, &user_agent)
|
||||
.send();
|
||||
|
||||
response = https_req.await.or(http_req.await).ok();
|
||||
response = tokio::select! {
|
||||
https_result = https_req => https_result.ok(),
|
||||
http_result = http_req => http_result.ok(),
|
||||
};
|
||||
|
||||
if response.is_some() {
|
||||
break;
|
||||
|
|
@ -50,36 +53,45 @@ pub async fn return_http_data(options: &LibOptions) -> HashMap<String, HttpData>
|
|||
}
|
||||
|
||||
if let Some(resp) = response {
|
||||
http_data.checked_host = host.clone();
|
||||
// Those are always set
|
||||
http_data.protocol = resp.url().scheme().to_string();
|
||||
http_data.status_code = resp.status().as_u16();
|
||||
http_data.final_url = resp.url().to_string();
|
||||
|
||||
if options.assign_response_data {
|
||||
http_data = assign_response_data(http_data, resp, options.return_filters).await;
|
||||
} else {
|
||||
http_data.status_code = resp.status().as_u16();
|
||||
http_data.http_status = "ACTIVE".to_string();
|
||||
if !from_cli {
|
||||
assign_response_data(&mut http_data, resp, options.return_filters).await;
|
||||
}
|
||||
} else {
|
||||
http_data.http_status = "INACTIVE".to_string();
|
||||
}
|
||||
|
||||
if !options.quiet_flag
|
||||
&& !http_data.checked_host.is_empty()
|
||||
&& !http_data.final_url.is_empty()
|
||||
&& (filter_codes.is_empty()
|
||||
|| filter_codes.contains(&http_data.status_code.to_string()))
|
||||
&& (exclude_codes.is_empty()
|
||||
|| !exclude_codes.contains(&http_data.status_code.to_string()))
|
||||
{
|
||||
if options.show_status_codes {
|
||||
println!(
|
||||
// Use faster I/O for high-throughput scenarios
|
||||
use std::io::{self, Write};
|
||||
let stdout = io::stdout();
|
||||
let mut handle = stdout.lock();
|
||||
|
||||
if options.show_full_data {
|
||||
let _ = writeln!(
|
||||
handle,
|
||||
"{},[{}],[{}]",
|
||||
http_data.checked_host, http_data.final_url, http_data.status_code
|
||||
);
|
||||
} else {
|
||||
println!("{},[{}]", http_data.checked_host, http_data.final_url);
|
||||
let _ = writeln!(
|
||||
handle,
|
||||
"{}://{}",
|
||||
http_data.protocol, http_data.checked_host
|
||||
);
|
||||
}
|
||||
}
|
||||
(host, http_data)
|
||||
(host.clone(), http_data)
|
||||
}
|
||||
}))
|
||||
.buffer_unordered(threads)
|
||||
|
|
@ -100,30 +112,33 @@ pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client {
|
|||
.redirect(policy)
|
||||
.danger_accept_invalid_certs(true)
|
||||
.use_native_tls()
|
||||
.pool_max_idle_per_host(50)
|
||||
.pool_idle_timeout(std::time::Duration::from_secs(30))
|
||||
.tcp_keepalive(std::time::Duration::from_secs(60))
|
||||
.build()
|
||||
.expect("Failed to create HTTP client")
|
||||
}
|
||||
|
||||
#[allow(clippy::field_reassign_with_default)]
|
||||
pub async fn assign_response_data(
|
||||
mut http_data: HttpData,
|
||||
resp: Response,
|
||||
return_filters: bool,
|
||||
) -> HttpData {
|
||||
pub async fn assign_response_data(http_data: &mut HttpData, resp: Response, return_filters: bool) {
|
||||
let headers = resp.headers().clone();
|
||||
let url = resp.url().clone();
|
||||
|
||||
http_data.http_status = "ACTIVE".to_string();
|
||||
http_data.status_code = resp.status().as_u16();
|
||||
http_data.final_url = url.to_string();
|
||||
http_data.protocol = url.scheme().to_string();
|
||||
http_data.content_type = headers
|
||||
.get(CONTENT_TYPE)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
|
||||
let full_body = (resp.text().await).unwrap_or_default();
|
||||
let full_body = {
|
||||
const MAX_BODY_SIZE: usize = 1024 * 1024; // 1MB limit
|
||||
match resp.text().await {
|
||||
Ok(text) if text.len() <= MAX_BODY_SIZE => text,
|
||||
Ok(text) => text.chars().take(MAX_BODY_SIZE).collect(),
|
||||
Err(_) => String::new(),
|
||||
}
|
||||
};
|
||||
|
||||
http_data.content_length = headers
|
||||
.get(CONTENT_LENGTH)
|
||||
|
|
@ -132,10 +147,14 @@ pub async fn assign_response_data(
|
|||
|
||||
http_data.headers = format!("{headers:?}");
|
||||
|
||||
return_title_and_body(&mut http_data, &full_body);
|
||||
return_title_and_body(http_data, &full_body);
|
||||
|
||||
http_data.words_count = full_body.split_whitespace().count();
|
||||
http_data.lines = full_body.lines().count() + 1;
|
||||
let lines_count = full_body.lines().count();
|
||||
let words_count = full_body.split_whitespace().count();
|
||||
|
||||
http_data.words_count = words_count;
|
||||
http_data.lines = lines_count + 1;
|
||||
http_data.points_to_another_host = url.host_str() != Some(&http_data.checked_host);
|
||||
|
||||
if return_filters {
|
||||
let host = url.host_str().unwrap_or_default();
|
||||
|
|
@ -143,42 +162,29 @@ pub async fn assign_response_data(
|
|||
let user_agents = utils::user_agents();
|
||||
http_data.bad_data = return_filters_data(host, client, user_agents).await;
|
||||
}
|
||||
|
||||
http_data
|
||||
}
|
||||
|
||||
pub fn return_title_and_body(http_data: &mut HttpData, body: &str) {
|
||||
let document = Html::parse_document(body);
|
||||
|
||||
// Return title
|
||||
match Selector::parse("title") {
|
||||
Ok(selector) => {
|
||||
if let Some(title_element) = document.select(&selector).next() {
|
||||
http_data.title = title_element.inner_html();
|
||||
} else {
|
||||
http_data.title = "NULL".to_string();
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("Failed to parse selector: {err:?}");
|
||||
}
|
||||
let title_selector = Selector::parse("title").ok();
|
||||
let body_selector = Selector::parse("body").ok();
|
||||
|
||||
if let Some(title_sel) = &title_selector {
|
||||
http_data.title = document
|
||||
.select(title_sel)
|
||||
.next()
|
||||
.map(|element| element.inner_html())
|
||||
.unwrap_or_else(|| "NULL".to_string());
|
||||
}
|
||||
|
||||
// Return body
|
||||
match Selector::parse("body") {
|
||||
Ok(selector) => {
|
||||
if let Some(body_element) = document.select(&selector).next() {
|
||||
http_data.body = body_element.inner_html();
|
||||
} else {
|
||||
http_data.body = "NULL".to_string();
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
eprintln!("Failed to parse selector: {err:?}");
|
||||
}
|
||||
if let Some(body_sel) = &body_selector {
|
||||
http_data.body = document
|
||||
.select(body_sel)
|
||||
.next()
|
||||
.map(|element| element.inner_html())
|
||||
.unwrap_or_else(|| "NULL".to_string());
|
||||
}
|
||||
|
||||
drop(document);
|
||||
}
|
||||
|
||||
pub async fn return_filters_data(
|
||||
|
|
@ -211,28 +217,23 @@ pub async fn return_filters_data(
|
|||
user_agents: user_agents_list,
|
||||
retries: 1,
|
||||
threads,
|
||||
assign_response_data: true,
|
||||
quiet_flag: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let data = return_http_data(&lib_options).await;
|
||||
let data = return_http_data(&lib_options, false).await;
|
||||
|
||||
data.values()
|
||||
.map(|http_data| {
|
||||
http_filters
|
||||
.bad_http_lengths
|
||||
.append(&mut vec![http_data.content_length.to_string()]);
|
||||
http_filters.bad_words_numbers.append(&mut vec![http_data
|
||||
.body
|
||||
.split(' ')
|
||||
.count()
|
||||
.to_string()]);
|
||||
http_filters
|
||||
.bad_lines_numbers
|
||||
.append(&mut vec![http_data.lines.to_string()]);
|
||||
})
|
||||
.for_each(drop);
|
||||
data.values().for_each(|http_data| {
|
||||
http_filters
|
||||
.bad_http_lengths
|
||||
.push(http_data.content_length.to_string());
|
||||
http_filters
|
||||
.bad_words_numbers
|
||||
.push(http_data.words_count.to_string());
|
||||
http_filters
|
||||
.bad_lines_numbers
|
||||
.push(http_data.lines.to_string());
|
||||
});
|
||||
|
||||
http_filters
|
||||
}
|
||||
|
|
|
|||
38
src/main.rs
38
src/main.rs
|
|
@ -10,19 +10,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
// Eval args
|
||||
let args = args::Cli::parse();
|
||||
|
||||
let filter_codes = args.filter_codes;
|
||||
let exclude_codes = args.exclude_codes;
|
||||
let threads = args.threads;
|
||||
let retries = args.retries;
|
||||
let timeout = args.timeout;
|
||||
let max_redirects = args.max_redirects;
|
||||
let user_agents_list = utils::user_agents();
|
||||
let show_status_codes = args.show_codes;
|
||||
|
||||
let client = httplib::return_http_client(timeout, max_redirects);
|
||||
|
||||
let mut buffer = String::new();
|
||||
|
||||
let mut hosts = HashSet::new();
|
||||
|
||||
if args.domain.is_some() {
|
||||
|
|
@ -39,29 +27,25 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|||
} else {
|
||||
io::stdin().read_to_string(&mut buffer).await?;
|
||||
hosts = buffer.lines().map(str::to_owned).collect();
|
||||
};
|
||||
}
|
||||
|
||||
let lib_options = LibOptions {
|
||||
hosts,
|
||||
client,
|
||||
user_agents: user_agents_list,
|
||||
retries,
|
||||
threads,
|
||||
filter_codes,
|
||||
exclude_codes,
|
||||
show_status_codes,
|
||||
client: httplib::return_http_client(args.timeout, args.max_redirects),
|
||||
user_agents: utils::user_agents(),
|
||||
retries: args.retries,
|
||||
threads: args.threads,
|
||||
filter_codes: args.filter_codes,
|
||||
exclude_codes: args.exclude_codes,
|
||||
show_full_data: args.show_full_data,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if !args.quiet {
|
||||
if show_status_codes {
|
||||
println!("DOMAIN,[FINAL_URL],[STATUS_CODE]");
|
||||
} else {
|
||||
println!("DOMAIN,[FINAL_URL]");
|
||||
}
|
||||
if !args.quiet && args.show_full_data {
|
||||
println!("DOMAIN,[FINAL_URL],[STATUS_CODE]");
|
||||
}
|
||||
|
||||
let _ = httplib::return_http_data(&lib_options).await;
|
||||
let _ = httplib::return_http_data(&lib_options, true).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,17 @@ pub struct HttpData {
|
|||
pub bad_data: HTTPFilters,
|
||||
pub html_file_path: String,
|
||||
pub screenshot_data: Vec<u8>,
|
||||
pub points_to_another_host: bool,
|
||||
}
|
||||
|
||||
impl HttpData {
|
||||
#[inline]
|
||||
pub fn new(host: String) -> Self {
|
||||
Self {
|
||||
checked_host: host,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
|
|
@ -37,7 +48,6 @@ pub struct LibOptions {
|
|||
pub return_filters: bool,
|
||||
pub filter_codes: Option<String>,
|
||||
pub exclude_codes: Option<String>,
|
||||
pub show_status_codes: bool,
|
||||
pub assign_response_data: bool,
|
||||
pub show_full_data: bool,
|
||||
pub quiet_flag: bool,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use rand::{seq::SliceRandom, thread_rng};
|
||||
use rand::{rng, seq::IndexedRandom};
|
||||
|
||||
#[must_use]
|
||||
pub fn user_agents() -> Vec<String> {
|
||||
|
|
@ -37,7 +37,7 @@ pub fn user_agents() -> Vec<String> {
|
|||
pub fn return_random_user_agent(strings: &[String]) -> String {
|
||||
let empty_string = String::new();
|
||||
strings
|
||||
.choose(&mut thread_rng())
|
||||
.choose(&mut rng())
|
||||
.unwrap_or(&empty_string)
|
||||
.to_string()
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue