diff --git a/Cargo.lock b/Cargo.lock index b9e1f99..14046aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "ansi_term" version = "0.11.0" @@ -9,6 +11,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "async-trait" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atty" version = "0.2.14" @@ -93,6 +106,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + [[package]] name = "encoding_rs" version = "0.8.28" @@ -102,6 +121,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c5f0096a91d210159eceb2ff5e1c4da18388a170e1e3ce948aac9c8fdbbf595" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fhc" version = "0.5.1" @@ -275,6 +306,15 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -284,6 +324,17 @@ dependencies = [ "libc", ] +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi", +] + [[package]] name = "http" version = "0.2.4" @@ -335,13 +386,28 @@ dependencies = [ "httpdate", "itoa", "pin-project", - "socket2", + "socket2 0.4.0", "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "futures-util", + "hyper", + "log", + "rustls", + "tokio", + "tokio-rustls", + "webpki", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -385,6 +451,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ipconfig" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e2f18aece9709094573a9f24f483c4f65caa4298e2f7ae1b71cc65d853fad7" +dependencies = [ + "socket2 0.3.19", + "widestring", + "winapi", + "winreg 0.6.2", +] + [[package]] name = "ipnet" version = "2.3.0" @@ -418,6 +496,12 @@ version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + [[package]] name = "lock_api" version = "0.4.4" @@ -436,6 +520,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + [[package]] name = "matches" version = "0.1.8" @@ -658,6 +757,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.9" @@ -739,6 +844,7 @@ dependencies = [ "http", "http-body", "hyper", + "hyper-rustls", "hyper-tls", "ipnet", "js-sys", @@ -748,15 +854,57 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "rustls", "serde", "serde_urlencoded", "tokio", "tokio-native-tls", + "tokio-rustls", + "trust-dns-resolver", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg", + "webpki-roots", + "winreg 0.7.0", +] + +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64", + "log", + "ring", + "sct", + "webpki", ] [[package]] @@ -781,6 +929,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.2.0" @@ -854,6 +1012,17 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if", + "libc", + "winapi", +] + [[package]] name = "socket2" version = "0.4.0" @@ -864,6 +1033,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "strsim" version = "0.8.0" @@ -904,6 +1079,26 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinyvec" version = "1.2.0" @@ -960,6 +1155,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls", + "tokio", + "webpki", +] + [[package]] name = "tokio-util" version = "0.6.7" @@ -1000,6 +1206,51 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "trust-dns-proto" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca94d4e9feb6a181c690c4040d7a24ef34018d8313ac5044a61d21222ae24e31" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "lazy_static", + "log", + "rand", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecae383baad9995efaa34ce8e57d12c3f305e545887472a492b838f4b5cfb77a" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lazy_static", + "log", + "lru-cache", + "parking_lot", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "trust-dns-proto", +] + [[package]] name = "try-lock" version = "0.2.3" @@ -1024,6 +1275,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" + [[package]] name = "unicode-width" version = "0.1.8" @@ -1036,6 +1293,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "url" version = "2.2.2" @@ -1154,6 +1417,31 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +dependencies = [ + "webpki", +] + +[[package]] +name = "widestring" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" + [[package]] name = "winapi" version = "0.3.9" @@ -1176,6 +1464,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9" +dependencies = [ + "winapi", +] + [[package]] name = "winreg" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 156f156..87f7804 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,14 @@ readme = "README.md" tokio = { version = "1.6.0", features = ["full", "io-util"] } futures = "0.3.15" clap = "2.33.3" -reqwest = { version = "0.11.3", features = ["default-tls"] } +reqwest = { version = "0.11.3", features = ["trust-dns", "rustls-tls", "native-tls"] } openssl = { version = "0.10.34", features = ["vendored"] } rand = "0.8.3" + + +[profile.release] +lto = true +codegen-units = 1 +panic = 'abort' +incremental = false +opt-level = "s" \ No newline at end of file diff --git a/src/httplib.rs b/src/httplib.rs new file mode 100644 index 0000000..b402214 --- /dev/null +++ b/src/httplib.rs @@ -0,0 +1,105 @@ +use std::collections::{HashMap, HashSet}; + +use reqwest::Client; + +use { + crate::{structs::HttpData, utils}, + futures::stream::StreamExt, + reqwest::{self, header::USER_AGENT}, +}; + +#[allow(clippy::too_many_arguments)] +pub async fn return_http_data( + hosts: HashSet, + client: Client, + user_agents_list: Vec, + retries: usize, + mut threads: usize, + conditional_response_code: u16, + show_status_codes: bool, + quiet_flag: bool, +) -> HashMap { + if hosts.len() < threads { + threads = hosts.len(); + } + + futures::stream::iter(hosts.into_iter().map(|host| { + // Use a random user agent + let user_agent = utils::return_random_string(&user_agents_list); + // HTTP/HTTP URLs + let https_url = format!("https://{}", host); + let http_url = format!("http://{}", host); + // Create futures + let https_send_fut = client.get(&https_url).header(USER_AGENT, &user_agent); + let http_send_fut = client.get(&http_url).header(USER_AGENT, &user_agent); + + async move { + let mut http_data = HttpData::default(); + + if retries != 1 { + let mut counter = 0; + while counter < retries { + if let Ok(resp) = https_send_fut + .try_clone() + .expect("Failed to clone https future") + .send() + .await + { + http_data.host_url = https_url.clone(); + http_data.status_code = resp.status().as_u16(); + http_data.http_status = "ACTIVE".to_string(); + drop(resp) + } else if let Ok(resp) = http_send_fut + .try_clone() + .expect("Failed to clone http future") + .send() + .await + { + http_data.host_url = http_url.clone(); + http_data.status_code = resp.status().as_u16(); + http_data.http_status = "ACTIVE".to_string(); + drop(resp) + } + counter += 1 + } + } else if let Ok(resp) = https_send_fut.send().await { + http_data.host_url = https_url; + http_data.status_code = resp.status().as_u16(); + http_data.http_status = "ACTIVE".to_string(); + drop(resp) + } else if let Ok(resp) = http_send_fut.send().await { + http_data.host_url = http_url; + http_data.status_code = resp.status().as_u16(); + http_data.http_status = "ACTIVE".to_string(); + drop(resp) + } else { + http_data.http_status = "INACTIVE".to_string(); + } + if !quiet_flag && (!http_data.host_url.is_empty() && conditional_response_code == 0) + || ((!http_data.host_url.is_empty() && conditional_response_code != 0) + && (http_data.status_code >= conditional_response_code + && http_data.status_code <= conditional_response_code + 99)) + { + if show_status_codes { + println!("{},{}", http_data.host_url, http_data.status_code) + } else { + println!("{}", http_data.host_url) + } + } + (host, http_data) + } + })) + .buffer_unordered(threads) + .collect::>() + .await +} + +pub fn return_http_client(timeout: u64) -> Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(timeout)) + .danger_accept_invalid_certs(true) + .trust_dns(true) + .use_native_tls() + .build() + .expect("Failed to create HTTP client") +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..739a5b1 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,3 @@ +pub mod httplib; +pub mod structs; +pub mod utils; diff --git a/src/main.rs b/src/main.rs index cab439b..8d62b7a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,10 @@ -mod utils; +use std::collections::HashSet; + +use fhc::httplib; use { clap::{value_t, App, Arg}, - futures::stream::StreamExt, - reqwest::{self, header::USER_AGENT}, + fhc::utils, tokio::{ self, io::{self, AsyncReadExt}, @@ -22,7 +23,7 @@ async fn main() -> Result<(), Box> { .short("t") .long("threads") .takes_value(true) - .help("Number of threads. Default: 100"), + .help("Number of threads. Default: 50"), ) .arg( Arg::with_name("timeout") @@ -102,16 +103,13 @@ async fn main() -> Result<(), Box> { } else { 0 }; - let threads = value_t!(matches.value_of("threads"), usize).unwrap_or_else(|_| 100); + let threads = value_t!(matches.value_of("threads"), usize).unwrap_or_else(|_| 50); let retries = value_t!(matches.value_of("retries"), usize).unwrap_or_else(|_| 1); - let timeout = value_t!(matches.value_of("timeout"), u64).unwrap_or_else(|_| 5); + let timeout = value_t!(matches.value_of("timeout"), u64).unwrap_or_else(|_| 3); let user_agents_list = utils::user_agents(); let show_status_codes = matches.is_present("show-codes"); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(timeout)) - .danger_accept_invalid_certs(true) - .build()?; + let client = httplib::return_http_client(timeout); // Read stdin @@ -119,7 +117,7 @@ async fn main() -> Result<(), Box> { let mut buffer = String::new(); stdin.read_to_string(&mut buffer).await?; - let hosts: Vec = if matches.is_present("domain") { + let hosts: HashSet = if matches.is_present("domain") { let domain = value_t!(matches, "domain", String).unwrap(); buffer .lines() @@ -129,67 +127,17 @@ async fn main() -> Result<(), Box> { buffer.lines().map(str::to_owned).collect() }; - futures::stream::iter(hosts.into_iter().map(|host| { - // Use a random user agent - let user_agent = utils::return_random_string(&user_agents_list); - // HTTP/HTTP URLs - let https_url = format!("https://{}", host); - let http_url = format!("http://{}", host); - // Create futures - let https_send_fut = client.get(&https_url).header(USER_AGENT, &user_agent); - let http_send_fut = client.get(&http_url).header(USER_AGENT, &user_agent); - - async move { - let mut response_code = 0; - let mut valid_url = String::new(); - if retries != 1 { - let mut counter = 0; - while counter < retries { - if let Ok(resp) = https_send_fut - .try_clone() - .expect("Failed to clone https future") - .send() - .await - { - valid_url = https_url.clone(); - response_code = resp.status().as_u16(); - drop(resp) - } else if let Ok(resp) = http_send_fut - .try_clone() - .expect("Failed to clone http future") - .send() - .await - { - valid_url = http_url.clone(); - response_code = resp.status().as_u16(); - drop(resp) - } - counter += 1 - } - } else if let Ok(resp) = https_send_fut.send().await { - valid_url = https_url; - response_code = resp.status().as_u16(); - drop(resp) - } else if let Ok(resp) = http_send_fut.send().await { - valid_url = http_url; - response_code = resp.status().as_u16(); - drop(resp) - } - if (!valid_url.is_empty() && conditional_response_code == 0) - || ((!valid_url.is_empty() && conditional_response_code != 0) - && (response_code >= conditional_response_code - && response_code <= conditional_response_code + 99)) - { - if show_status_codes { - println!("{},{}", valid_url, response_code) - } else { - println!("{}", valid_url) - } - } - } - })) - .buffer_unordered(threads) - .collect::>() + httplib::return_http_data( + hosts, + client, + user_agents_list, + retries, + threads, + conditional_response_code, + show_status_codes, + false, + ) .await; + Ok(()) } diff --git a/src/structs.rs b/src/structs.rs new file mode 100644 index 0000000..b629fb9 --- /dev/null +++ b/src/structs.rs @@ -0,0 +1,16 @@ +#[derive(Clone)] +pub struct HttpData { + pub http_status: String, + pub host_url: String, + pub status_code: u16, +} + +impl Default for HttpData { + fn default() -> Self { + HttpData { + http_status: String::from(""), + host_url: String::from(""), + status_code: 0, + } + } +}