edu4rdshl/fhc
1
0
Fork 0
mirror of https://github.com/edu4rdshl/fhc.git synced 2026-07-17 23:24:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Performance improvements.

Browse source 
Signed-off-by: Eduard Tolosa <edu4rdshl@protonmail.com>
This commit is contained in:
Eduard Tolosa 2022-04-11 23:46:47 -05:00
parent 0965d04087
commit df80de9263
No known key found for this signature in database
GPG key ID: 3A574A4009F553E5
1 changed files with 68 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

108
src/httplib.rs
View file

@ -8,7 +8,7 @@ use reqwest::{
Client, Response, Client, Response,
}; };
use crate::structs::HTTPFilters; use crate::structs::{HTTPFilters, LibOptions};
use { use {
crate::{structs::HttpData, utils}, crate::{structs::HttpData, utils},
@ -19,67 +19,88 @@ use {
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
#[async_recursion] #[async_recursion]
pub async fn return_http_data( pub async fn return_http_data(options: &LibOptions) -> HashMap<String, HttpData> {
hosts: HashSet<String>, let threads = if options.hosts.len() < options.threads {
client: Client, options.hosts.len()
user_agents_list: Vec<String>, } else {
retries: usize, options.threads
mut threads: usize, };
return_filers: bool,
conditional_response_code: u16,
show_status_codes: bool,
quiet_flag: bool,
) -> HashMap<String, HttpData> {
if hosts.len() < threads {
threads = hosts.len();
}
futures::stream::iter(hosts.into_iter().map(|host| { futures::stream::iter(options.hosts.clone().into_iter().map(|host| {
// Use a random user agent // Use a random user agent
let user_agent = utils::return_random_string(&user_agents_list); let user_agent = utils::return_random_string(&options.user_agents);
// HTTP/HTTP URLs // HTTP/HTTP URLs
let https_url = format!("https://{}", host); let https_url = format!("https://{}", host);
let http_url = format!("http://{}", host); let http_url = format!("http://{}", host);
// Create futures // Create futures
let https_send_fut = client.get(&https_url).header(USER_AGENT, &user_agent); let https_send_fut = options
let http_send_fut = client.get(&http_url).header(USER_AGENT, &user_agent); .client
.get(&https_url)
.header(USER_AGENT, &user_agent);
let http_send_fut = options
.client
.get(&http_url)
.header(USER_AGENT, &user_agent);
let mut http_data = HttpData::default(); let mut http_data = HttpData::default();
let mut is_http = false;
let mut response = Option::<Response>::None;
async move { async move {
if retries != 1 { if options.retries != 1 {
let mut counter = 0; let mut counter = 0;
while counter < retries { while counter < options.retries {
if let Ok(resp) = https_send_fut if let Ok(resp) = https_send_fut
.try_clone() .try_clone()
.expect("Failed to clone https future") .expect("Failed to clone https future")
.send() .send()
.await .await
{ {
http_data = assign_response_data(resp, return_filers).await; response = Some(resp);
break;
} else if let Ok(resp) = http_send_fut } else if let Ok(resp) = http_send_fut
.try_clone() .try_clone()
.expect("Failed to clone http future") .expect("Failed to clone http future")
.send() .send()
.await .await
{ {
http_data = assign_response_data(resp, return_filers).await; is_http = true;
response = Some(resp);
break;
} }
counter += 1 counter += 1
} }
} else if let Ok(resp) = https_send_fut.send().await { } else if let Ok(resp) = https_send_fut.send().await {
http_data = assign_response_data(resp, return_filers).await; response = Some(resp);
} else if let Ok(resp) = http_send_fut.send().await { } else if let Ok(resp) = http_send_fut.send().await {
http_data = assign_response_data(resp, return_filers).await; is_http = true;
response = Some(resp);
}
match response {
Some(resp) => {
if options.assign_response_data {
http_data =
assign_response_data(http_data, resp, options.return_filters).await;
} else { } else {
http_data.host_url = if is_http { http_url } else { https_url.clone() };
http_data.status_code = resp.status().as_u16();
http_data.http_status = "ACTIVE".to_string();
};
}
None => {
http_data.http_status = "INACTIVE".to_string(); http_data.http_status = "INACTIVE".to_string();
} }
if !quiet_flag && (!http_data.host_url.is_empty() && conditional_response_code == 0) };
|| ((!http_data.host_url.is_empty() && conditional_response_code != 0)
&& (http_data.status_code >= conditional_response_code if !options.quiet_flag
&& http_data.status_code <= conditional_response_code + 99)) && (!http_data.host_url.is_empty() && options.conditional_response_code == 0)
|| ((!http_data.host_url.is_empty() && options.conditional_response_code != 0)
&& (http_data.status_code >= options.conditional_response_code
&& http_data.status_code <= options.conditional_response_code + 99))
{ {
if show_status_codes { if options.show_status_codes {
println!("{},{}", http_data.host_url, http_data.status_code) println!("{},{}", http_data.host_url, http_data.status_code)
} else { } else {
println!("{}", http_data.host_url) println!("{}", http_data.host_url)
@ -105,9 +126,12 @@ pub fn return_http_client(timeout: u64, max_redirects: usize) -> Client {
} }
#[allow(clippy::field_reassign_with_default)] #[allow(clippy::field_reassign_with_default)]
pub async fn assign_response_data(resp: Response, return_filers: bool) -> HttpData { pub async fn assign_response_data(
mut http_data: HttpData,
resp: Response,
return_filers: bool,
) -> HttpData {
let headers = resp.headers().clone(); let headers = resp.headers().clone();
let mut http_data = HttpData::default();
let mut url = resp.url().to_owned(); let mut url = resp.url().to_owned();
http_data.http_status = "ACTIVE".to_string(); http_data.http_status = "ACTIVE".to_string();
@ -214,18 +238,18 @@ pub async fn return_filters_data(
let threads = urls_to_check.len(); let threads = urls_to_check.len();
let mut http_filters = HTTPFilters::default(); let mut http_filters = HTTPFilters::default();
let data = return_http_data(
urls_to_check, let lib_options = LibOptions {
hosts: urls_to_check,
client, client,
user_agents_list, user_agents: user_agents_list,
1, retries: 1,
threads, threads,
false, quiet_flag: true,
0, ..Default::default()
false, };
true,
) let data = return_http_data(&lib_options).await;
.await;
data.iter() data.iter()
.map(|(_, http_data)| { .map(|(_, http_data)| {