Feat/html to markdown #106 (#115)

* feat: add HTML-to-Markdown conversion for web content - Add readabilityrs for content extraction - Add html-to-markdown for conversion - Feature-gated behind html-markdown flag - Integrates with HTTP tool response handling - Includes comprehensive tests and examples Closes #106 * Update comments for is_html_response helper and fix tests to not fail silently in certain instances --------- Co-authored-by: Zach Frederick <[email protected]> Co-authored-by: Illia Polosukhin <[email protected]>
2026-08-26 15:40:18 +00:00 · 2026-02-21 06:05:16 +00:00
parent 436066415b
commit dbd3e0807f
16 changed files with 20726 additions and 4 deletions
@@ -0,0 +1,128 @@
+//! HTML to Markdown conversion for HTTP responses.
+//!
+//! Two-stage pipeline: readability (extract article) -> html-to-markdown-rs (convert to md).
+//! When the `html-to-markdown` feature is disabled, passthrough only.
+
+use crate::tools::tool::ToolError;
+
+#[cfg(feature = "html-to-markdown")]
+use html_to_markdown_rs::convert;
+#[cfg(feature = "html-to-markdown")]
+use readabilityrs::Readability;
+
+#[cfg(not(feature = "html-to-markdown"))]
+pub fn convert_html_to_markdown(html: &str, _url: &str) -> Result<String, ToolError> {
+    Ok(html.to_string())
+}
+
+#[cfg(feature = "html-to-markdown")]
+pub fn convert_html_to_markdown(html: &str, url: &str) -> Result<String, ToolError> {
+    let readability = Readability::new(html, Some(url), None)
+        .map_err(|e| ToolError::ExecutionFailed(format!("readability parser: {:?}", e)))?;
+
+    let article = readability.parse().ok_or_else(|| {
+        ToolError::ExecutionFailed("failed to extract article content".to_string())
+    })?;
+
+    let clean_html = article.content.ok_or_else(|| {
+        ToolError::ExecutionFailed("no content extracted from article".to_string())
+    })?;
+
+    let markdown = convert(&clean_html, None)
+        .map_err(|e| ToolError::ExecutionFailed(format!("HTML to markdown: {}", e)))?;
+
+    Ok(markdown)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[cfg(not(feature = "html-to-markdown"))]
+    #[test]
+    fn passthrough_returns_input_unchanged_when_feature_disabled() {
+        {
+            let html = "<html><body>raw</body></html>";
+            let out = convert_html_to_markdown(html, "https://example.com/").unwrap();
+            assert_eq!(out, html);
+        }
+    }
+
+    #[cfg(not(feature = "html-to-markdown"))]
+    #[test]
+    fn passthrough_ignores_url_when_feature_disabled() {
+        {
+            let html = "anything";
+            let _ = convert_html_to_markdown(html, "").unwrap();
+            let _ = convert_html_to_markdown(html, "https://example.com/page").unwrap();
+        }
+    }
+
+    #[cfg(feature = "html-to-markdown")]
+    #[test]
+    fn simple_article_extracted_and_converted_to_markdown() {
+        // Readability needs enough content (default char_threshold ~500) and clear main content.
+        let html = r#"<!DOCTYPE html>
+<html><head><title>Test</title></head><body>
+<nav><a href="/">Home</a></nav>
+<main>
+  <article>
+    <h1>Test Title</h1>
+    <p>First paragraph with enough text so that readability's scoring finds this as the main content block. We need to exceed the default character threshold.</p>
+    <p>Second paragraph. More body text here to make the article clearly the dominant content area versus the short nav and footer.</p>
+    <p>Third paragraph for good measure. The extraction algorithm scores candidates by paragraph count and text length; this block should win.</p>
+  </article>
+</main>
+<footer><p>Footer</p></footer>
+</body></html>"#;
+        let out = convert_html_to_markdown(html, "https://example.com/article").unwrap();
+        assert!(
+            out.contains("Test Title"),
+            "expected title in output: {}",
+            out
+        );
+        assert!(
+            out.contains("First paragraph"),
+            "expected content in output: {}",
+            out
+        );
+        assert!(
+            out.contains("Second paragraph"),
+            "expected content in output: {}",
+            out
+        );
+        assert!(
+            !out.contains("<article>"),
+            "expected markdown, not raw HTML"
+        );
+    }
+
+    #[cfg(feature = "html-to-markdown")]
+    #[test]
+    fn returns_execution_error_on_empty_html() {
+        let result = convert_html_to_markdown("", "https://example.com/");
+        let err = result.unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            msg.contains("Execution failed") || msg.contains("extract") || msg.contains("content"),
+            "{}",
+            msg
+        );
+    }
+
+    #[cfg(feature = "html-to-markdown")]
+    #[test]
+    fn returns_execution_error_on_plain_text_not_html() {
+        let result = convert_html_to_markdown("not html at all", "https://example.com/");
+        let err = result.unwrap_err();
+        let msg = err.to_string();
+        assert!(
+            msg.contains("Execution failed")
+                || msg.contains("extract")
+                || msg.contains("content")
+                || msg.contains("parser"),
+            "{}",
+            msg
+        );
+    }
+}
@@ -15,6 +15,9 @@ use crate::secrets::SecretsStore;
 use crate::tools::tool::{ApprovalRequirement, Tool, ToolError, ToolOutput, require_str};
 use crate::tools::wasm::{InjectedCredentials, SharedCredentialRegistry, inject_credential};

+#[cfg(feature = "html-to-markdown")]
+use crate::tools::builtin::convert_html_to_markdown;
+
 /// Maximum response body size (5 MB).
 ///
 /// 5 MB is large enough for typical JSON API responses and moderate HTML pages,
@@ -126,6 +129,16 @@ fn is_disallowed_ip(ip: &IpAddr) -> bool {
    }
 }

+#[cfg(feature = "html-to-markdown")]
+/// Heuristic: treat as HTML if the `Content-Type` header contains `text/html`.
+fn is_html_response(headers: &HashMap<String, String>) -> bool {
+    headers
+        .iter()
+        .find(|(k, _)| k.eq_ignore_ascii_case("content-type"))
+        .map(|(_, v)| v.to_lowercase().contains("text/html"))
+        .unwrap_or(false)
+}
+
 fn parse_headers_param(
    headers: Option<&serde_json::Value>,
 ) -> Result<Vec<(String, String)>, ToolError> {
@@ -395,6 +408,19 @@ impl Tool for HttpTool {

        let body_text = String::from_utf8_lossy(&body_bytes).into_owned();

+        #[cfg(feature = "html-to-markdown")]
+        let body_text = if is_html_response(&headers) {
+            match convert_html_to_markdown(&body_text, parsed_url.as_str()) {
+                Ok(md) => md,
+                Err(e) => {
+                    tracing::warn!(url = %parsed_url, error = %e, "HTML-to-markdown conversion failed, returning raw HTML");
+                    body_text
+                }
+            }
+        } else {
+            body_text
+        };
+
        // Try to parse as JSON, fall back to string
        let body: serde_json::Value = serde_json::from_str(&body_text)
            .unwrap_or_else(|_| serde_json::Value::String(body_text.clone()));
@@ -30,3 +30,7 @@ pub use routine::{
 pub use shell::ShellTool;
 pub use skill_tools::{SkillInstallTool, SkillListTool, SkillRemoveTool, SkillSearchTool};
 pub use time::TimeTool;
+
+mod html_converter;
+
+pub use html_converter::convert_html_to_markdown;