use std::collections::{HashMap, HashSet};
use std::sync::Arc;

use anyhow::{bail, Result};
use console::style;
use futures::future;
use lazy_static::lazy_static;
use scraper::{Html, Selector};
use uuid::Uuid;

use crate::filters::{SimilarityFilter, WildcardFilter};
use crate::message::FeroxMessage;
use crate::scanner::RESPONSES;
use crate::{
    config::OutputLevel,
    event_handlers::{Command, Handles},
    progress::PROGRESS_PRINTER,
    response::FeroxResponse,
    skip_fail,
    url::FeroxUrl,
    utils::{ferox_print, fmt_err, logged_request},
    COMMON_FILE_EXTENSIONS, DEFAULT_BACKUP_EXTENSIONS, DEFAULT_METHOD,
};

lazy_static! {
    /// Pre-built HashSet of file extensions for O(1) lookup in directory listing detection
    /// Combines COMMON_FILE_EXTENSIONS and DEFAULT_BACKUP_EXTENSIONS
    static ref FILE_EXTENSION_SET: HashSet<&'static str> = {
        let mut set = HashSet::with_capacity(
            COMMON_FILE_EXTENSIONS.len() + DEFAULT_BACKUP_EXTENSIONS.len()
        );
        for ext in COMMON_FILE_EXTENSIONS.iter() {
            set.insert(*ext);
        }
        for ext in DEFAULT_BACKUP_EXTENSIONS.iter() {
            set.insert(*ext);
        }
        set
    };
}

/// enum representing the different servers that `parse_html` can detect when directory listing is
/// enabled
#[derive(Copy, Debug, Clone)]
pub enum DirListingType {
    /// apache server, detected by `Index of /`
    Apache,

    /// tomcat/python server, detected by `Directory Listing for /`
    TomCatOrPython,

    /// ASP.NET server, detected by `Directory Listing -- /`
    AspDotNet,

    /// custom/non-standard directory listing, detected by high-signal heuristics
    Custom,

    // /// IIS/Azure server, detected by `HOST_NAME - /` (not currently used)
    // IIS_AZURE,
    /// variant that represents the absence of directory listing
    None,
}

/// Wrapper around the results of running a directory listing detection against a target web page
#[derive(Debug, Clone)]
pub struct DirListingResult {
    /// type of server where directory listing was detected
    /// i.e. https://portswigger.net/kb/issues/00600100_directory-listing
    pub dir_list_type: Option<DirListingType>,

    /// the `FeroxResponse` generated during detection
    pub response: FeroxResponse,
}

/// wrapper around the results of running a wildcard detection against a target web page
#[derive(Copy, Debug, Clone)]
pub enum WildcardResult {
    /// variant that represents a wildcard directory
    WildcardDirectory(usize),

    /// variant that represents the presence of a 404-like response
    FourOhFourLike(usize),
}

/// container for heuristics related info
pub struct HeuristicTests {
    /// Handles object for event handler interaction
    handles: Arc<Handles>,
}

/// HeuristicTests implementation
impl HeuristicTests {
    /// create a new HeuristicTests struct
    pub fn new(handles: Arc<Handles>) -> Self {
        Self { handles }
    }

    /// Simple helper to return a uuid, formatted as lowercase without hyphens
    ///
    /// `length` determines the number of uuids to string together. Each uuid
    /// is 32 characters long. So, a length of 1 return a 32 character string,
    /// a length of 2 returns a 64 character string, and so on...
    fn unique_string(&self, length: usize) -> String {
        log::trace!("enter: unique_string({length})");
        let mut ids = vec![];

        for _ in 0..length {
            ids.push(Uuid::new_v4().as_simple().to_string());
        }

        let unique_id = ids.join("");

        log::trace!("exit: unique_string -> {unique_id}");
        unique_id
    }

    /// Simply tries to connect to all given sites before starting to scan
    ///
    /// In the event that no sites can be reached, the program will exit.
    ///
    /// Any urls that are found to be alive are returned to the caller.
    pub async fn connectivity(&self, target_urls: &[String]) -> Result<Vec<String>> {
        log::trace!("enter: connectivity_test({target_urls:?})");

        let mut good_urls = vec![];

        for target_url in target_urls {
            let url = FeroxUrl::from_string(target_url, self.handles.clone());
            let request = skip_fail!(url.format("", None));

            let result = logged_request(&request, DEFAULT_METHOD, None, self.handles.clone()).await;

            match result {
                Ok(_) => {
                    good_urls.push(target_url.to_owned());
                }
                Err(e) => {
                    if matches!(
                        self.handles.config.output_level,
                        OutputLevel::Default | OutputLevel::Quiet
                    ) {
                        if e.to_string().contains(":SSL") {
                            let msg = format!("Could not connect to {target_url} due to {} errors (run with {} to ignore), skipping...\n  => {}\n",style("SSL").red(), style("--insecure").yellow().bright(), e.root_cause());
                            ferox_print(&msg, &PROGRESS_PRINTER);
                        } else {
                            ferox_print(
                                &format!(
                                    "Could not connect to {target_url}, skipping...\n  => {}\n",
                                    e.root_cause()
                                ),
                                &PROGRESS_PRINTER,
                            );
                        }
                    }
                    log::warn!("{e}");
                }
            }
        }

        if good_urls.is_empty() {
            bail!("Could not connect to any target provided");
        }

        log::trace!("exit: connectivity_test -> {good_urls:?}");
        Ok(good_urls)
    }

    /// heuristic designed to detect when a server has directory listing enabled
    pub async fn directory_listing(&self, target_url: &str) -> Result<Option<DirListingResult>> {
        log::trace!("enter: directory_listing({target_url})");

        let tgt = if !target_url.ends_with('/') {
            // if left unchanged, this function would be called against redirects that point to
            // valid directories for most, if not all, directories beyond the initial urls.
            // so, instead of `directory_listing("http://localhost") -> None` we get
            // `directory_listing("http://localhost/") -> Some(DirListingResult)` if there is
            // directory listing beyond the redirect
            format!("{target_url}/")
        } else {
            target_url.to_string()
        };

        let url = FeroxUrl::from_string(&tgt, self.handles.clone());
        let request = url.format("", None)?;

        let result = logged_request(&request, DEFAULT_METHOD, None, self.handles.clone()).await?;

        let ferox_response = FeroxResponse::from(
            result,
            &url.target,
            DEFAULT_METHOD,
            self.handles.config.output_level,
            self.handles.config.response_size_limit,
        )
        .await;

        let body = ferox_response.text();
        let html = Html::parse_document(body);

        if let Some(dir_type) = self.detect_directory_listing(&html) {
            // folks that run things and step away/rely on logs need to be notified of directory
            // listing, since they won't see the message on the bar; bastardizing FeroxMessage
            // for ease of implementation. This could use a bit of polish at some point.

            let msg = format!(
                "detected directory listing: {} ({:?})",
                target_url, dir_type
            );
            let ferox_msg = FeroxMessage {
                kind: "log".to_string(),
                message: msg.clone(),
                level: "MSG".to_string(),
                time_offset: 0.0,
                module: "feroxbuster::heuristics".to_string(),
            };
            self.handles
                .output
                .tx_file
                .send(Command::WriteToDisk(Box::new(ferox_msg)))
                .unwrap_or_default();

            log::info!("{msg}");

            let result = DirListingResult {
                dir_list_type: Some(dir_type),
                response: ferox_response,
            };

            log::trace!("exit: directory_listing -> {result:?}");
            return Ok(Some(result));
        }

        log::trace!("exit: directory_listing -> None");
        Ok(None)
    }

    /// Directory listing heuristic detection, uses <title> tag to make its determination. When
    /// the inner html of <title> matches one of the following, a `DirListingType` is returned.
    /// - apache: `Index of /`
    /// - tomcat/python: `Directory Listing for /`
    /// - ASP.NET: `Directory Listing -- /`
    /// - <host> - /: iis, azure, skipping due to loose heuristic
    /// - custom: detected by combining multiple high-signal heuristics
    fn detect_directory_listing(&self, html: &Html) -> Option<DirListingType> {
        log::trace!("enter: detect_directory_listing(html body...)");

        let title_selector = Selector::parse("title").ok()?;

        for t in html.select(&title_selector) {
            let title = t.inner_html().to_lowercase();

            let dirlist_type = if title.contains("directory listing for /") {
                Some(DirListingType::TomCatOrPython)
            } else if title.contains("index of /") {
                Some(DirListingType::Apache)
            } else if title.contains("directory listing -- /") {
                Some(DirListingType::AspDotNet)
            } else {
                // IIS_AZURE purposely skipped for now
                None
            };

            if dirlist_type.is_some() {
                log::trace!("exit: detect_directory_listing -> {dirlist_type:?}");
                return dirlist_type;
            }
        }

        // If no standard title-based detection, try high-signal custom heuristics
        let has_parent_link = self.has_parent_directory_link(html);
        let has_table_headers = self.has_directory_table_headers(html);
        let has_sorting_params = self.has_sorting_query_params(html);
        let has_link_density = self.has_high_link_density(html);

        let signal_count = [
            has_parent_link,
            has_table_headers,
            has_sorting_params,
            has_link_density,
        ]
        .iter()
        .filter(|&&x| x)
        .count();

        if signal_count >= 2 {
            let mut signals = Vec::new();
            if has_parent_link {
                signals.push("parent-link");
            }
            if has_table_headers {
                signals.push("table-headers");
            }
            if has_sorting_params {
                signals.push("sorting-params");
            }
            if has_link_density {
                signals.push("link-density");
            }
            log::debug!("custom directory listing signals: [{}]", signals.join(", "));
            log::trace!("exit: detect_directory_listing -> Some(Custom)");
            return Some(DirListingType::Custom);
        }

        log::trace!("exit: detect_directory_listing -> None");
        None
    }

    /// check if the HTML contains a link to the parent directory
    ///
    /// returns true if any anchor element has:
    /// - href equals "../" or ".."
    /// - visible text contains "parent directory", "to parent", or "up to parent"
    fn has_parent_directory_link(&self, html: &Html) -> bool {
        log::trace!("enter: has_parent_directory_link");

        let Some(anchor_selector) = Selector::parse("a").ok() else {
            log::warn!("failed to parse anchor selector in has_parent_directory_link");
            return false;
        };

        for anchor in html.select(&anchor_selector) {
            if let Some(href) = anchor.value().attr("href") {
                let href_lower = href.trim().to_lowercase();
                if href_lower == "../" || href_lower == ".." {
                    log::trace!("exit: has_parent_directory_link -> true (href match)");
                    return true;
                }
            }

            let text = anchor.text().collect::<String>().to_lowercase();
            let text_trimmed = text.trim();
            if text_trimmed.contains("parent directory")
                || text_trimmed.contains("to parent")
                || text_trimmed.contains("up to parent")
            {
                log::trace!("exit: has_parent_directory_link -> true (text match)");
                return true;
            }
        }

        log::trace!("exit: has_parent_directory_link -> false");
        false
    }

    /// check if the HTML contains table headers typical of directory listings
    ///
    /// returns true if at least two of the following header categories are present:
    /// - name headers: "file name", "filename", "name"
    /// - size headers: "size", "file size"
    /// - time headers: "date", "last modified", "modified", "last mod"
    fn has_directory_table_headers(&self, html: &Html) -> bool {
        log::trace!("enter: has_directory_table_headers");

        let Some(th_selector) = Selector::parse("th").ok() else {
            log::warn!("failed to parse th selector in has_directory_table_headers");
            return false;
        };
        let Some(td_selector) = Selector::parse("td").ok() else {
            log::warn!("failed to parse td selector in has_directory_table_headers");
            return false;
        };

        let mut headers = Vec::new();

        // try <th> elements first
        for th in html.select(&th_selector) {
            let text = th.text().collect::<String>().to_lowercase();
            headers.push(text.trim().to_string());
        }

        // fallback: if no <th> elements, try first row of <td> elements
        if headers.is_empty() {
            if let Ok(tr_selector) = Selector::parse("tr") {
                if let Some(first_row) = html.select(&tr_selector).next() {
                    for td in first_row.select(&td_selector) {
                        let text = td.text().collect::<String>().to_lowercase();
                        headers.push(text.trim().to_string());
                    }
                }
            }
        }

        let mut has_name = false;
        let mut has_size = false;
        let mut has_time = false;

        for header in headers {
            if header == "name" || header.contains("file name") || header.contains("filename") {
                has_name = true;
            }
            if header.contains("size") || header.contains("file size") {
                has_size = true;
            }
            if header.contains("date")
                || header.contains("last modified")
                || header.contains("modified")
                || header.contains("last mod")
            {
                has_time = true;
            }
        }

        let category_count = [has_name, has_size, has_time]
            .iter()
            .filter(|&&x| x)
            .count();
        let result = category_count >= 2;

        log::trace!("exit: has_directory_table_headers -> {result}");
        result
    }

    /// check if the HTML contains sorting query parameters typical of auto-index pages
    ///
    /// returns true if any anchor href contains sorting parameters like:
    /// - ?C=N (name), ?C=S (size), ?C=M (modified), ?C=D (date)
    /// - optionally combined with &O=A or &O=D (ascending/descending)
    fn has_sorting_query_params(&self, html: &Html) -> bool {
        log::trace!("enter: has_sorting_query_params");

        let Some(anchor_selector) = Selector::parse("a").ok() else {
            log::warn!("failed to parse anchor selector in has_sorting_query_params");
            return false;
        };

        for anchor in html.select(&anchor_selector) {
            if let Some(href) = anchor.value().attr("href") {
                let href_lower = href.to_lowercase();
                if href_lower.contains("?c=n")
                    || href_lower.contains("?c=s")
                    || href_lower.contains("?c=m")
                    || href_lower.contains("?c=d")
                {
                    log::trace!("exit: has_sorting_query_params -> true");
                    return true;
                }
            }
        }

        log::trace!("exit: has_sorting_query_params -> false");
        false
    }

    /// check if the HTML has a high density of file/directory links
    ///
    /// returns true if there are at least 3 links that look like files or directories:
    /// - href ends with '/' (likely subdirectory)
    /// - href looks like a file (common extensions)
    fn has_high_link_density(&self, html: &Html) -> bool {
        log::trace!("enter: has_high_link_density");

        const MIN_LINKS: usize = 3;

        let Some(anchor_selector) = Selector::parse("a").ok() else {
            log::warn!("failed to parse anchor selector in has_high_link_density");
            return false;
        };
        let mut count = 0;

        for anchor in html.select(&anchor_selector) {
            if let Some(href) = anchor.value().attr("href") {
                let href_trimmed = href.trim();

                // skip parent directory links and fragments
                if href_trimmed == "../" || href_trimmed == ".." || href_trimmed.starts_with('#') {
                    continue;
                }

                // check if it's a directory (ends with /)
                if href_trimmed.ends_with('/') {
                    count += 1;
                    continue;
                }

                // check if it looks like a file - extract extension and O(1) lookup
                let href_lower = href_trimmed.to_lowercase();
                if let Some(dot_pos) = href_lower.rfind('.') {
                    let extension = &href_lower[dot_pos..];
                    if FILE_EXTENSION_SET.contains(extension) {
                        count += 1;
                    }
                }
            }
        }

        let result = count >= MIN_LINKS;
        log::trace!("exit: has_high_link_density -> {result} (count: {count})");
        result
    }

    /// given a target's base url, attempt to automatically detect its 404 response
    /// pattern(s), and then set filters that will exclude those patterns from future
    /// responses
    pub async fn detect_404_like_responses(
        &self,
        target_url: &str,
    ) -> Result<Option<WildcardResult>> {
        log::trace!("enter: detect_404_like_responses({target_url:?})");

        if self.handles.config.dont_filter {
            // early return, dont_filter scans don't need tested
            log::trace!("exit: detect_404_like_responses -> dont_filter is true");
            return Ok(None);
        }

        let mut req_counter = 0;

        let data = if self.handles.config.data.is_empty() {
            None
        } else {
            Some(self.handles.config.data.as_slice())
        };

        // To take care of slash when needed
        let slash = if self.handles.config.add_slash {
            Some("/")
        } else {
            None
        };

        // no matter what, we want an empty extension for the base case
        let mut extensions = vec!["".to_string()];

        // and then we want to add any extensions that was specified
        // or has since been added to the running config
        for ext in &self.handles.config.extensions {
            extensions.push(format!(".{ext}"));
        }

        // for every method, attempt to id its 404 response
        //
        // a good example of one where the GET/POST differ is on hackthebox:
        // - http://prd.m.rendering-api.interface.htb/api
        //
        // a good example of one where the heuristics return a 403 and a 404 (apache)
        // as well as return two different types of 404s based on the file extension
        // - http://10.10.11.198 (Encoding box in normal labs)
        //
        // both methods and extensions can elicit different responses from a given
        // server, so both are considered when building auto-filter rules
        for method in self.handles.config.methods.iter() {
            for extension in extensions.iter() {
                // build out the 6 paths we'll use
                let paths = [
                    ("", 1),
                    ("", 3),
                    (".htaccess", 1),
                    (".htaccess", 3),
                    ("admin", 1),
                    ("admin", 3),
                ]
                .map(|(prefix, length)| {
                    format!("{prefix}{}{extension}", self.unique_string(length))
                });

                // allow all 6 requests to fly asynchronously
                let responses = future::join_all(paths.into_iter().map(|path| async move {
                    let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());

                    let Ok(nonexistent_url) = ferox_url.format(&path, slash) else {
                        return None;
                    };

                    // example requests:
                    // - http://localhost/2fc1077836ad43ab98b7a31c2ca28fea
                    // - http://localhost/92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
                    // - http://localhost/.htaccessa005a2131e68449aa26e99029c914c09
                    // - http://localhost/.htaccess92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
                    // - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280
                    // - http://localhost/admin92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
                    let Ok(response) =
                        logged_request(&nonexistent_url, method, data, self.handles.clone()).await
                    else {
                        return None;
                    };

                    if !self
                        .handles
                        .config
                        .status_codes
                        .contains(&response.status().as_u16())
                    {
                        // if the response code isn't one that's accepted via -s values, then skip to the next
                        //
                        // the default value for -s is all status codes, so unless the user says otherwise
                        // this won't fire
                        return None;
                    }

                    Some(
                        FeroxResponse::from(
                            response,
                            &ferox_url.target,
                            method,
                            self.handles.config.output_level,
                            self.handles.config.response_size_limit,
                        )
                        .await,
                    )
                }))
                .await // await gives vector of options containing feroxresponses
                .into_iter()
                .flatten() // strip out the none values
                .collect::<Vec<_>>();

                if responses.len() < 2 {
                    // don't have enough responses to make a determination, continue to next method
                    log::debug!("not enough responses to make a determination");
                    continue;
                }

                // check the responses for similarities on which we can filter, multiple may be returned
                let Some((wildcard_filters, wildcard_responses)) =
                    self.examine_404_like_responses(&responses)
                else {
                    // no match was found during analysis of responses
                    log::warn!("no match found for 404 responses");
                    continue;
                };

                // report to the user, if appropriate
                if matches!(
                    self.handles.config.output_level,
                    OutputLevel::Default | OutputLevel::Quiet
                ) {
                    // sentry value to control whether or not to print the filter
                    // used because we only want to print the same filter once
                    let mut print_sentry;

                    if let Ok(filters) = self.handles.filters.data.filters.read() {
                        for new_wildcard in &wildcard_filters {
                            // reset the sentry for every new wildcard produced by examine_404_like_responses
                            print_sentry = true;

                            for other in filters.iter() {
                                if let Some(other_wildcard) =
                                    other.as_any().downcast_ref::<WildcardFilter>()
                                {
                                    // check the new wildcard against all existing wildcards, if it was added
                                    // on the cli or by a previous directory, don't print it
                                    if new_wildcard.as_ref() == other_wildcard {
                                        print_sentry = false;
                                        break;
                                    }
                                }
                            }

                            // if we're here, we've found a new wildcard that we didn't previously display, print it
                            if print_sentry {
                                ferox_print(&format!("{new_wildcard}"), &PROGRESS_PRINTER);
                            }
                        }
                    }
                }

                // create the new filter
                for wildcard in wildcard_filters {
                    self.handles.filters.send(Command::AddFilter(wildcard))?;
                }

                // if we're here, we've detected a 404-like response pattern, and we're already filtering for size/word/line
                //
                // in addition, we'll create a similarity filter as a fallback
                for resp in wildcard_responses {
                    let sim_filter = SimilarityFilter::from(resp);

                    self.handles
                        .filters
                        .send(Command::AddFilter(Box::new(sim_filter)))?;

                    if resp.is_directory() {
                        // response is either a 3XX with a Location header that matches url + '/'
                        // or it's a 2XX that ends with a '/'
                        // or it's a 403 that ends with a '/'

                        // set the wildcard flag to true, so we can check it when preventing
                        // recursion in event_handlers/scans.rs

                        // we'd need to clone the response to give ownership to the global list anyway
                        // so we'll also use that clone to set the wildcard flag
                        let mut cloned_resp = resp.clone();

                        cloned_resp.set_wildcard(true);

                        // add the response to the global list of responses
                        RESPONSES.insert(cloned_resp);

                        // function-internal magic number, indicates that we've detected a wildcard directory
                        req_counter += 100;
                    }
                }
            }
        }

        log::trace!("exit: detect_404_like_responses");

        let retval = if req_counter >= 100 {
            WildcardResult::WildcardDirectory(req_counter)
        } else {
            WildcardResult::FourOhFourLike(req_counter)
        };

        Ok(Some(retval))
    }

    /// for all responses, group them by status code, then examine chars/words/lines.
    /// if all responses' respective lengths within a status code grouping match
    /// each other, we can assume that will remain true for subsequent non-existent urls
    ///
    /// within a status code grouping, values are examined from most to
    /// least specific (content length, word count, line count)
    #[allow(clippy::vec_box)] // the box is needed in the caller and i dont feel like changing it
    fn examine_404_like_responses<'a>(
        &self,
        responses: &'a [FeroxResponse],
    ) -> Option<(Vec<Box<WildcardFilter>>, Vec<&'a FeroxResponse>)> {
        // aside from word/line/byte counts, additional discriminators are status code
        // extension, and request method. The request method and extension are handled by
        // the caller, since they're part of the request and make up the nested for loops
        // in detect_404_like_responses.
        //
        // The status code is handled here, since it's part of the response to catch cases
        // where we have something like a 403 and a 404

        let mut size_sentry = true;
        let mut word_sentry = true;
        let mut line_sentry = true;

        // returned vec of boxed wildcard filters
        let mut wildcards = Vec::new();

        // returned vec of ferox responses that are needed for additional
        // analysis
        let mut wild_responses = Vec::new();

        // mapping of grouped responses to status code
        let mut grouped_responses = HashMap::new();

        // iterate over all responses and add each response to its
        // corresponding status code group
        for response in responses {
            grouped_responses
                .entry(response.status())
                .or_insert_with(Vec::new)
                .push(response);
        }

        // iterate over each grouped response and determine the most specific
        // filter that can be applied to all responses in the group, i.e.
        // start from byte count and work 'out' to line count
        for response_group in grouped_responses.values() {
            if response_group.len() < 2 {
                // not enough responses to make a determination
                continue;
            }

            let method = response_group[0].method();
            let status_code = response_group[0].status();
            let content_length = response_group[0].content_length();
            let word_count = response_group[0].word_count();
            let line_count = response_group[0].line_count();

            for response in &response_group[1..] {
                // if any of the responses differ in length, that particular
                // response length type is no longer a candidate for filtering
                if response.content_length() != content_length {
                    size_sentry = false;
                }

                if response.word_count() != word_count {
                    word_sentry = false;
                }

                if response.line_count() != line_count {
                    line_sentry = false;
                }
            }

            if !size_sentry && !word_sentry && !line_sentry {
                // none of the response lengths match, so we can't filter on any of them
                continue;
            }

            let mut wildcard = WildcardFilter {
                content_length: None,
                line_count: None,
                word_count: None,
                method: method.to_string(),
                status_code: status_code.as_u16(),
                dont_filter: self.handles.config.dont_filter,
            };

            match (size_sentry, word_sentry, line_sentry) {
                (true, true, true) => {
                    // all three types of length match, so we can't filter on any of them
                    wildcard.content_length = Some(content_length);
                    wildcard.word_count = Some(word_count);
                    wildcard.line_count = Some(line_count);
                }
                (true, true, false) => {
                    // content length and word count match, so we can filter on either
                    wildcard.content_length = Some(content_length);
                    wildcard.word_count = Some(word_count);
                }
                (true, false, true) => {
                    // content length and line count match, so we can filter on either
                    wildcard.content_length = Some(content_length);
                    wildcard.line_count = Some(line_count);
                }
                (false, true, true) => {
                    // word count and line count match, so we can filter on either
                    wildcard.word_count = Some(word_count);
                    wildcard.line_count = Some(line_count);
                }
                (true, false, false) => {
                    // content length matches, so we can filter on that
                    wildcard.content_length = Some(content_length);
                }
                (false, true, false) => {
                    // word count matches, so we can filter on that
                    wildcard.word_count = Some(word_count);
                }
                (false, false, true) => {
                    // line count matches, so we can filter on that
                    wildcard.line_count = Some(line_count);
                }
                (false, false, false) => {
                    // none of the length types match, so we can't filter on any of them
                    unreachable!("no wildcard size matches; handled by the if statement above");
                }
            };

            wild_responses.push(response_group[0]);
            wildcards.push(Box::new(wildcard));
        }

        Some((wildcards, wild_responses))
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    /// request a unique string of 32bytes * a value returns correct result
    fn heuristics_unique_string_returns_correct_length() {
        let (handles, _) = Handles::for_testing(None, None);
        let tester = HeuristicTests::new(Arc::new(handles));
        for i in 0..10 {
            assert_eq!(tester.unique_string(i).len(), i * 32);
        }
    }

    #[test]
    /// `detect_directory_listing` correctly identifies tomcat/python instances
    fn detect_directory_listing_finds_tomcat_python() {
        let html = "<title>directory listing for /</title>";
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        let dirlist_type = heuristics.detect_directory_listing(&parsed);
        assert!(matches!(
            dirlist_type.unwrap(),
            DirListingType::TomCatOrPython
        ));
    }

    #[test]
    /// `detect_directory_listing` correctly identifies apache instances
    fn detect_directory_listing_finds_apache() {
        let html = "<title>index of /</title>";
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        let dirlist_type = heuristics.detect_directory_listing(&parsed);
        assert!(matches!(dirlist_type.unwrap(), DirListingType::Apache));
    }

    #[test]
    /// `detect_directory_listing` correctly identifies ASP.NET instances
    fn detect_directory_listing_finds_asp_dot_net() {
        let html = "<title>directory listing -- /</title>";
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        let dirlist_type = heuristics.detect_directory_listing(&parsed);
        assert!(matches!(dirlist_type.unwrap(), DirListingType::AspDotNet));
    }

    #[test]
    /// `detect_directory_listing` returns None when heuristic doesn't match
    fn detect_directory_listing_returns_none_as_default() {
        let html = "<title>derp listing -- /</title>";
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        let dirlist_type = heuristics.detect_directory_listing(&parsed);
        assert!(dirlist_type.is_none());
    }

    #[test]
    /// `has_parent_directory_link` detects parent directory links by href
    fn has_parent_directory_link_detects_by_href() {
        let html = r#"<a href="../">Go up</a>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(heuristics.has_parent_directory_link(&parsed));

        let html2 = r#"<a href="..">Go up</a>"#;
        let parsed2 = Html::parse_document(html2);
        assert!(heuristics.has_parent_directory_link(&parsed2));
    }

    #[test]
    /// `has_parent_directory_link` detects parent directory links by text
    fn has_parent_directory_link_detects_by_text() {
        let html = r#"<a href="/parent">Parent Directory</a>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(heuristics.has_parent_directory_link(&parsed));

        let html2 = r#"<a href="/up">To Parent</a>"#;
        let parsed2 = Html::parse_document(html2);
        assert!(heuristics.has_parent_directory_link(&parsed2));
    }

    #[test]
    /// `has_parent_directory_link` returns false when no parent link
    fn has_parent_directory_link_returns_false_when_absent() {
        let html = r#"<a href="/about">About</a>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(!heuristics.has_parent_directory_link(&parsed));
    }

    #[test]
    /// `has_directory_table_headers` detects table headers with name and size
    fn has_directory_table_headers_detects_name_and_size() {
        let html = r#"<table><thead><tr><th>File Name</th><th>Size</th></tr></thead></table>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(heuristics.has_directory_table_headers(&parsed));
    }

    #[test]
    /// `has_directory_table_headers` detects table headers with name and date
    fn has_directory_table_headers_detects_name_and_date() {
        let html = r#"<table><thead><tr><th>Name</th><th>Last Modified</th></tr></thead></table>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(heuristics.has_directory_table_headers(&parsed));
    }

    #[test]
    /// `has_directory_table_headers` returns false with only one category
    fn has_directory_table_headers_requires_two_categories() {
        let html = r#"<table><thead><tr><th>Name</th><th>Description</th></tr></thead></table>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(!heuristics.has_directory_table_headers(&parsed));
    }

    #[test]
    /// `has_sorting_query_params` detects Apache-style sorting parameters
    fn has_sorting_query_params_detects_apache_style() {
        let html = r#"<a href="?C=N&O=A">Name</a><a href="?C=S&O=D">Size</a>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(heuristics.has_sorting_query_params(&parsed));
    }

    #[test]
    /// `has_sorting_query_params` returns false when no sorting params
    fn has_sorting_query_params_returns_false_when_absent() {
        let html = r#"<a href="/page?q=search">Search</a>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(!heuristics.has_sorting_query_params(&parsed));
    }

    #[test]
    /// `has_high_link_density` detects high density of file/directory links
    fn has_high_link_density_detects_files_and_dirs() {
        let html = r#"
            <a href="backup/">backup/</a>
            <a href="file1.html">file1.html</a>
            <a href="file2.txt">file2.txt</a>
        "#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(heuristics.has_high_link_density(&parsed));
    }

    #[test]
    /// `has_high_link_density` requires at least 3 links
    fn has_high_link_density_requires_minimum_links() {
        let html = r#"
            <a href="backup/">backup/</a>
            <a href="file.html">file.html</a>
        "#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(!heuristics.has_high_link_density(&parsed));
    }

    #[test]
    /// `has_high_link_density` ignores parent directory links
    fn has_high_link_density_ignores_parent_links() {
        let html = r#"
            <a href="../">Parent</a>
            <a href="backup/">backup/</a>
            <a href="file.html">file.html</a>
        "#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        assert!(!heuristics.has_high_link_density(&parsed));
    }

    #[test]
    /// `detect_directory_listing` detects custom listing with 2+ signals
    fn detect_directory_listing_detects_custom_with_multiple_signals() {
        // This HTML has parent link, table headers, sorting params, and link density
        let html = r#"
            <table><thead><tr>
                <th><a href="?C=N&O=A">File Name</a></th>
                <th><a href="?C=S&O=A">Size</a></th>
            </tr></thead>
            <tbody>
                <tr><td><a href="../">Parent directory/</a></td></tr>
                <tr><td><a href="backup/">backup/</a></td></tr>
                <tr><td><a href="pass.html">pass.html</a></td></tr>
            </tbody></table>
        "#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        let dirlist_type = heuristics.detect_directory_listing(&parsed);
        assert!(matches!(dirlist_type, Some(DirListingType::Custom)));
    }

    #[test]
    /// `detect_directory_listing` requires at least 2 signals for custom detection
    fn detect_directory_listing_requires_two_signals() {
        // This HTML has only parent link (1 signal)
        let html = r#"<a href="../">Parent directory/</a>"#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        let dirlist_type = heuristics.detect_directory_listing(&parsed);
        assert!(dirlist_type.is_none());
    }

    #[test]
    /// `detect_directory_listing` detects Root-Me sample page as custom
    fn detect_directory_listing_detects_rootme_sample() {
        // Simplified version of response.html from Root-Me
        let html = r#"
            <table id="list">
                <thead><tr>
                    <th><a href="?C=N&O=A">File Name</a></th>
                    <th><a href="?C=S&O=A">File Size</a></th>
                    <th><a href="?C=M&O=A">Date</a></th>
                </tr></thead>
                <tbody>
                    <tr><td><a href="../">Parent directory/</a></td><td>-</td><td>-</td></tr>
                    <tr><td><a href="backup/">backup/</a></td><td>-</td><td>2021-Dec-10</td></tr>
                    <tr><td><a href="pass.html">pass.html</a></td><td>346 B</td><td>2021-Dec-10</td></tr>
                </tbody>
            </table>
        "#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        let dirlist_type = heuristics.detect_directory_listing(&parsed);
        assert!(matches!(dirlist_type, Some(DirListingType::Custom)));
    }

    #[test]
    /// `detect_directory_listing` does not trigger on pages with many random links
    fn detect_directory_listing_ignores_generic_pages() {
        let html = r#"
            <nav>
                <a href="/about">About</a>
                <a href="/contact">Contact</a>
                <a href="/services">Services</a>
                <a href="/products">Products</a>
            </nav>
        "#;
        let parsed = Html::parse_document(html);
        let handles = Handles::for_testing(None, None);
        let heuristics = HeuristicTests::new(Arc::new(handles.0));
        let dirlist_type = heuristics.detect_directory_listing(&parsed);
        assert!(dirlist_type.is_none());
    }
}
