mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 07:20:19 +00:00
* feat: add HTML-to-Markdown conversion for web content - Add readabilityrs for content extraction - Add html-to-markdown for conversion - Feature-gated behind html-markdown flag - Integrates with HTTP tool response handling - Includes comprehensive tests and examples Closes #106 * Update comments for is_html_response helper and fix tests to not fail silently in certain instances --------- Co-authored-by: Zach Frederick <[email protected]> Co-authored-by: Illia Polosukhin <[email protected]>
This commit is contained in:
Zach Frederick
co-authored by
Zach Frederick
Illia Polosukhin
Zach Frederick
Illia Polosukhin
parent
436066415b
commit
dbd3e0807f
@@ -0,0 +1 @@
|
||||
tests/test-pages/**/*.html linguist-generated=true
|
||||
Generated
+434
-3
@@ -70,6 +70,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"const-random",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy 0.8.37",
|
||||
@@ -194,6 +195,15 @@ version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063"
|
||||
|
||||
[[package]]
|
||||
name = "astral-tl"
|
||||
version = "0.7.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d90933ffb0f97e2fc2e0de21da9d3f20597b804012d199843a6fe7c2810d28f3"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-broadcast"
|
||||
version = "0.7.2"
|
||||
@@ -930,6 +940,26 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-random"
|
||||
version = "0.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
|
||||
dependencies = [
|
||||
"const-random-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-random-macro"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"once_cell",
|
||||
"tiny-keccak",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.4.2"
|
||||
@@ -1105,6 +1135,21 @@ dependencies = [
|
||||
"target-lexicon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
|
||||
dependencies = [
|
||||
"crc-catalog",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc-catalog"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
@@ -1250,6 +1295,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crunchy"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
@@ -1261,6 +1312,29 @@ dependencies = [
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2"
|
||||
dependencies = [
|
||||
"cssparser-macros",
|
||||
"dtoa-short",
|
||||
"itoa",
|
||||
"phf 0.13.1",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cssparser-macros"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctr"
|
||||
version = "0.9.2"
|
||||
@@ -1503,12 +1577,33 @@ version = "0.15.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"
|
||||
|
||||
[[package]]
|
||||
name = "dtoa-short"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
|
||||
dependencies = [
|
||||
"dtoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "ego-tree"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
@@ -1708,6 +1803,12 @@ version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
|
||||
|
||||
[[package]]
|
||||
name = "foldhash"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
@@ -1759,6 +1860,16 @@ version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
|
||||
|
||||
[[package]]
|
||||
name = "futf"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
|
||||
dependencies = [
|
||||
"mac",
|
||||
"new_debug_unreachable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.31"
|
||||
@@ -1899,6 +2010,15 @@ dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getopts"
|
||||
version = "0.2.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
|
||||
dependencies = [
|
||||
"unicode-width 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.17"
|
||||
@@ -2017,7 +2137,7 @@ version = "0.15.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
|
||||
dependencies = [
|
||||
"foldhash",
|
||||
"foldhash 0.1.5",
|
||||
"serde",
|
||||
]
|
||||
|
||||
@@ -2026,6 +2146,11 @@ name = "hashbrown"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"equivalent",
|
||||
"foldhash 0.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
@@ -2081,6 +2206,54 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html-escape"
|
||||
version = "0.2.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
|
||||
dependencies = [
|
||||
"utf8-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html-to-markdown-rs"
|
||||
version = "2.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb31d75f2fdbc8d889d78a912e10c22c30451afb44ee3310f5bfcabf79a31a17"
|
||||
dependencies = [
|
||||
"ahash 0.8.12",
|
||||
"astral-tl",
|
||||
"base64 0.22.1",
|
||||
"html-escape",
|
||||
"html5ever 0.38.0",
|
||||
"lru",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6452c4751a24e1b99c3260d505eaeee76a050573e61f30ac2c924ddc7236f01e"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever 0.36.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "html5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1054432bae2f14e0061e33d23402fbaa67a921d319d56adc6bcf887ddad1cbc2"
|
||||
dependencies = [
|
||||
"log",
|
||||
"markup5ever 0.38.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.12"
|
||||
@@ -2528,6 +2701,7 @@ dependencies = [
|
||||
"fs4",
|
||||
"futures",
|
||||
"hkdf",
|
||||
"html-to-markdown-rs",
|
||||
"http-body-util",
|
||||
"hyper 1.8.1",
|
||||
"hyper-util",
|
||||
@@ -2538,6 +2712,7 @@ dependencies = [
|
||||
"postgres-types",
|
||||
"pretty_assertions",
|
||||
"rand 0.8.5",
|
||||
"readabilityrs",
|
||||
"refinery",
|
||||
"regex",
|
||||
"reqwest",
|
||||
@@ -2657,6 +2832,21 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "kuchikikiki"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b73885c6a3cefdf7a1db0327cefbe4b9b72cac94cae4b19ede4fa492d8af02a0"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"crc",
|
||||
"cssparser",
|
||||
"html5ever 0.38.0",
|
||||
"indexmap 2.13.0",
|
||||
"precomputed-hash",
|
||||
"selectors 0.35.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy-regex"
|
||||
version = "3.5.1"
|
||||
@@ -2807,7 +2997,7 @@ dependencies = [
|
||||
"log",
|
||||
"memchr",
|
||||
"phf 0.11.3",
|
||||
"phf_codegen",
|
||||
"phf_codegen 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
"uncased",
|
||||
]
|
||||
@@ -2901,12 +3091,27 @@ version = "0.4.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "lru"
|
||||
version = "0.16.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593"
|
||||
dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "mac"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "mach2"
|
||||
version = "0.4.3"
|
||||
@@ -2916,6 +3121,28 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.36.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c3294c4d74d0742910f8c7b466f44dda9eb2d5742c1e430138df290a1e8451c"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril 0.4.3",
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.38.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8983d30f2915feeaaab2d6babdd6bc7e9ed1a00b66b5e6d74df19aa9c0e91862"
|
||||
dependencies = [
|
||||
"log",
|
||||
"tendril 0.5.0",
|
||||
"web_atoms",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.2.0"
|
||||
@@ -3056,6 +3283,12 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
|
||||
|
||||
[[package]]
|
||||
name = "nibble_vec"
|
||||
version = "0.1.0"
|
||||
@@ -3426,6 +3659,7 @@ version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.13.1",
|
||||
"serde",
|
||||
]
|
||||
@@ -3436,10 +3670,20 @@ version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_generator 0.11.3",
|
||||
"phf_shared 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_codegen"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
@@ -3450,6 +3694,29 @@ dependencies = [
|
||||
"rand 0.8.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared 0.13.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
@@ -3613,6 +3880,12 @@ dependencies = [
|
||||
"zerocopy 0.8.37",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "precomputed-hash"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
|
||||
|
||||
[[package]]
|
||||
name = "pretty_assertions"
|
||||
version = "1.4.1"
|
||||
@@ -3880,6 +4153,24 @@ dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "readabilityrs"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3eb174b0af6c181a87d68b42800806657bfbdf88b566f819aaadb9d2a7b7699d"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"kuchikikiki",
|
||||
"once_cell",
|
||||
"regex",
|
||||
"scraper",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 1.0.69",
|
||||
"url",
|
||||
"v_htmlescape",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.3.5"
|
||||
@@ -4422,6 +4713,21 @@ version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
|
||||
|
||||
[[package]]
|
||||
name = "scraper"
|
||||
version = "0.25.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93cecd86d6259499c844440546d02f55f3e17bd286e529e48d1f9f67e92315cb"
|
||||
dependencies = [
|
||||
"cssparser",
|
||||
"ego-tree",
|
||||
"getopts",
|
||||
"html5ever 0.36.1",
|
||||
"precomputed-hash",
|
||||
"selectors 0.33.0",
|
||||
"tendril 0.4.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "seahash"
|
||||
version = "4.1.0"
|
||||
@@ -4493,6 +4799,44 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.33.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feef350c36147532e1b79ea5c1f3791373e61cbd9a6a2615413b3807bb164fb7"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"log",
|
||||
"new_debug_unreachable",
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"precomputed-hash",
|
||||
"rustc-hash 2.1.1",
|
||||
"servo_arc",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.35.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fdfed56cd634f04fe8b9ddf947ae3dc493483e819593d2ba17df9ad05db8b2"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"cssparser",
|
||||
"derive_more",
|
||||
"log",
|
||||
"new_debug_unreachable",
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"precomputed-hash",
|
||||
"rustc-hash 2.1.1",
|
||||
"servo_arc",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.27"
|
||||
@@ -4631,6 +4975,15 @@ dependencies = [
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "servo_arc"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_yml"
|
||||
version = "0.0.12"
|
||||
@@ -4800,6 +5153,30 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f42444fea5b87a39db4218d9422087e66a85d0e7a0963a439b07bcdf91804006"
|
||||
|
||||
[[package]]
|
||||
name = "string_cache"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"parking_lot",
|
||||
"phf_shared 0.13.1",
|
||||
"precomputed-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "string_cache_codegen"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
|
||||
dependencies = [
|
||||
"phf_generator 0.13.1",
|
||||
"phf_shared 0.13.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stringprep"
|
||||
version = "0.1.5"
|
||||
@@ -4967,6 +5344,27 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
|
||||
dependencies = [
|
||||
"futf",
|
||||
"mac",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tendril"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4790fc369d5a530f4b544b094e31388b9b3a37c0f4652ade4505945f5660d24"
|
||||
dependencies = [
|
||||
"new_debug_unreachable",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.4.1"
|
||||
@@ -5110,6 +5508,15 @@ dependencies = [
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiny-keccak"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
|
||||
dependencies = [
|
||||
"crunchy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.2"
|
||||
@@ -5751,6 +6158,12 @@ version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
|
||||
|
||||
[[package]]
|
||||
name = "utf8-width"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
@@ -5775,6 +6188,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "v_htmlescape"
|
||||
version = "0.15.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e8257fbc510f0a46eb602c10215901938b5c2a7d5e70fc11483b1d3c9b5b18c"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.1"
|
||||
@@ -6310,6 +6729,18 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web_atoms"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57a9779e9f04d2ac1ce317aee707aa2f6b773afba7b931222bff6983843b1576"
|
||||
dependencies = [
|
||||
"phf 0.13.1",
|
||||
"phf_codegen 0.13.1",
|
||||
"string_cache",
|
||||
"string_cache_codegen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "4.4.2"
|
||||
|
||||
+10
-1
@@ -149,6 +149,10 @@ bytes = "1"
|
||||
base64 = "0.22.1"
|
||||
mime_guess = "2.0.5"
|
||||
|
||||
# HTML to Markdown conversion (feature gated)
|
||||
html-to-markdown-rs = { version = "2.3", optional = true }
|
||||
readabilityrs = { version = "0.1.2", optional = true }
|
||||
|
||||
# macOS keychain
|
||||
[target.'cfg(target_os = "macos")'.dependencies]
|
||||
security-framework = "3"
|
||||
@@ -166,7 +170,7 @@ pretty_assertions = "1"
|
||||
tempfile = "3"
|
||||
|
||||
[features]
|
||||
default = ["postgres", "libsql"]
|
||||
default = ["postgres", "libsql", "html-to-markdown"]
|
||||
postgres = [
|
||||
"dep:deadpool-postgres",
|
||||
"dep:tokio-postgres",
|
||||
@@ -177,6 +181,11 @@ postgres = [
|
||||
]
|
||||
libsql = ["dep:libsql"]
|
||||
integration = []
|
||||
html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"]
|
||||
|
||||
[[test]]
|
||||
name = "html_to_markdown"
|
||||
required-features = ["html-to-markdown"]
|
||||
|
||||
# The profile that 'cargo dist' will build with
|
||||
[profile.dist]
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
//! HTML to Markdown conversion for HTTP responses.
|
||||
//!
|
||||
//! Two-stage pipeline: readability (extract article) -> html-to-markdown-rs (convert to md).
|
||||
//! When the `html-to-markdown` feature is disabled, passthrough only.
|
||||
|
||||
use crate::tools::tool::ToolError;
|
||||
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
use html_to_markdown_rs::convert;
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
use readabilityrs::Readability;
|
||||
|
||||
#[cfg(not(feature = "html-to-markdown"))]
|
||||
pub fn convert_html_to_markdown(html: &str, _url: &str) -> Result<String, ToolError> {
|
||||
Ok(html.to_string())
|
||||
}
|
||||
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
pub fn convert_html_to_markdown(html: &str, url: &str) -> Result<String, ToolError> {
|
||||
let readability = Readability::new(html, Some(url), None)
|
||||
.map_err(|e| ToolError::ExecutionFailed(format!("readability parser: {:?}", e)))?;
|
||||
|
||||
let article = readability.parse().ok_or_else(|| {
|
||||
ToolError::ExecutionFailed("failed to extract article content".to_string())
|
||||
})?;
|
||||
|
||||
let clean_html = article.content.ok_or_else(|| {
|
||||
ToolError::ExecutionFailed("no content extracted from article".to_string())
|
||||
})?;
|
||||
|
||||
let markdown = convert(&clean_html, None)
|
||||
.map_err(|e| ToolError::ExecutionFailed(format!("HTML to markdown: {}", e)))?;
|
||||
|
||||
Ok(markdown)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[cfg(not(feature = "html-to-markdown"))]
|
||||
#[test]
|
||||
fn passthrough_returns_input_unchanged_when_feature_disabled() {
|
||||
{
|
||||
let html = "<html><body>raw</body></html>";
|
||||
let out = convert_html_to_markdown(html, "https://example.com/").unwrap();
|
||||
assert_eq!(out, html);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "html-to-markdown"))]
|
||||
#[test]
|
||||
fn passthrough_ignores_url_when_feature_disabled() {
|
||||
{
|
||||
let html = "anything";
|
||||
let _ = convert_html_to_markdown(html, "").unwrap();
|
||||
let _ = convert_html_to_markdown(html, "https://example.com/page").unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
#[test]
|
||||
fn simple_article_extracted_and_converted_to_markdown() {
|
||||
// Readability needs enough content (default char_threshold ~500) and clear main content.
|
||||
let html = r#"<!DOCTYPE html>
|
||||
<html><head><title>Test</title></head><body>
|
||||
<nav><a href="/">Home</a></nav>
|
||||
<main>
|
||||
<article>
|
||||
<h1>Test Title</h1>
|
||||
<p>First paragraph with enough text so that readability's scoring finds this as the main content block. We need to exceed the default character threshold.</p>
|
||||
<p>Second paragraph. More body text here to make the article clearly the dominant content area versus the short nav and footer.</p>
|
||||
<p>Third paragraph for good measure. The extraction algorithm scores candidates by paragraph count and text length; this block should win.</p>
|
||||
</article>
|
||||
</main>
|
||||
<footer><p>Footer</p></footer>
|
||||
</body></html>"#;
|
||||
let out = convert_html_to_markdown(html, "https://example.com/article").unwrap();
|
||||
assert!(
|
||||
out.contains("Test Title"),
|
||||
"expected title in output: {}",
|
||||
out
|
||||
);
|
||||
assert!(
|
||||
out.contains("First paragraph"),
|
||||
"expected content in output: {}",
|
||||
out
|
||||
);
|
||||
assert!(
|
||||
out.contains("Second paragraph"),
|
||||
"expected content in output: {}",
|
||||
out
|
||||
);
|
||||
assert!(
|
||||
!out.contains("<article>"),
|
||||
"expected markdown, not raw HTML"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
#[test]
|
||||
fn returns_execution_error_on_empty_html() {
|
||||
let result = convert_html_to_markdown("", "https://example.com/");
|
||||
let err = result.unwrap_err();
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("Execution failed") || msg.contains("extract") || msg.contains("content"),
|
||||
"{}",
|
||||
msg
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
#[test]
|
||||
fn returns_execution_error_on_plain_text_not_html() {
|
||||
let result = convert_html_to_markdown("not html at all", "https://example.com/");
|
||||
let err = result.unwrap_err();
|
||||
let msg = err.to_string();
|
||||
assert!(
|
||||
msg.contains("Execution failed")
|
||||
|| msg.contains("extract")
|
||||
|| msg.contains("content")
|
||||
|| msg.contains("parser"),
|
||||
"{}",
|
||||
msg
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,9 @@ use crate::secrets::SecretsStore;
|
||||
use crate::tools::tool::{ApprovalRequirement, Tool, ToolError, ToolOutput, require_str};
|
||||
use crate::tools::wasm::{InjectedCredentials, SharedCredentialRegistry, inject_credential};
|
||||
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
use crate::tools::builtin::convert_html_to_markdown;
|
||||
|
||||
/// Maximum response body size (5 MB).
|
||||
///
|
||||
/// 5 MB is large enough for typical JSON API responses and moderate HTML pages,
|
||||
@@ -126,6 +129,16 @@ fn is_disallowed_ip(ip: &IpAddr) -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
/// Heuristic: treat as HTML if the `Content-Type` header contains `text/html`.
|
||||
fn is_html_response(headers: &HashMap<String, String>) -> bool {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case("content-type"))
|
||||
.map(|(_, v)| v.to_lowercase().contains("text/html"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn parse_headers_param(
|
||||
headers: Option<&serde_json::Value>,
|
||||
) -> Result<Vec<(String, String)>, ToolError> {
|
||||
@@ -395,6 +408,19 @@ impl Tool for HttpTool {
|
||||
|
||||
let body_text = String::from_utf8_lossy(&body_bytes).into_owned();
|
||||
|
||||
#[cfg(feature = "html-to-markdown")]
|
||||
let body_text = if is_html_response(&headers) {
|
||||
match convert_html_to_markdown(&body_text, parsed_url.as_str()) {
|
||||
Ok(md) => md,
|
||||
Err(e) => {
|
||||
tracing::warn!(url = %parsed_url, error = %e, "HTML-to-markdown conversion failed, returning raw HTML");
|
||||
body_text
|
||||
}
|
||||
}
|
||||
} else {
|
||||
body_text
|
||||
};
|
||||
|
||||
// Try to parse as JSON, fall back to string
|
||||
let body: serde_json::Value = serde_json::from_str(&body_text)
|
||||
.unwrap_or_else(|_| serde_json::Value::String(body_text.clone()));
|
||||
|
||||
@@ -30,3 +30,7 @@ pub use routine::{
|
||||
pub use shell::ShellTool;
|
||||
pub use skill_tools::{SkillInstallTool, SkillListTool, SkillRemoveTool, SkillSearchTool};
|
||||
pub use time::TimeTool;
|
||||
|
||||
mod html_converter;
|
||||
|
||||
pub use html_converter::convert_html_to_markdown;
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
//! Integration tests for HTML-to-Markdown conversion.
|
||||
//!
|
||||
//! For each directory in tests/test-pages/, loads source.html, runs the converter,
|
||||
//! and optionally verifies against expected.md and metadata.json (contains).
|
||||
//! Run with: cargo test --test html_to_markdown -- --nocapture
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Default, serde::Deserialize)]
|
||||
#[serde(default)]
|
||||
struct PageMetadata {
|
||||
/// If false, skip golden-file comparison even when expected.md exists.
|
||||
check_expected: Option<bool>,
|
||||
/// Strings that must each appear in the converted markdown.
|
||||
contains: Option<Vec<String>>,
|
||||
/// Base URL for readability. If omitted, use default test-pages URL.
|
||||
url: Option<String>,
|
||||
}
|
||||
|
||||
fn normalize(s: &str) -> String {
|
||||
let s = s.replace("\r\n", "\n");
|
||||
let s = s.trim();
|
||||
let lines: Vec<&str> = s.lines().map(|l| l.trim()).collect();
|
||||
lines.join("\n").trim_end().to_string()
|
||||
}
|
||||
|
||||
/// Normalize typographic/smart punctuation to ASCII so tests match converter output
|
||||
/// regardless of apostrophe/quote variants (e.g. U+2019 ' → U+0027 ').
|
||||
fn normalize_smart_punctuation(s: &str) -> String {
|
||||
s.replace('\u{2019}', "'") // RIGHT SINGLE QUOTATION MARK
|
||||
.replace('\u{2018}', "'") // LEFT SINGLE QUOTATION MARK
|
||||
.replace('\u{201C}', "\"") // LEFT DOUBLE QUOTATION MARK
|
||||
.replace('\u{201D}', "\"") // RIGHT DOUBLE QUOTATION MARK
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn convert_test_pages_to_markdown() {
|
||||
let test_pages = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests")
|
||||
.join("test-pages");
|
||||
|
||||
let entries =
|
||||
std::fs::read_dir(&test_pages).expect("test-pages directory not found or not readable");
|
||||
|
||||
let mut converted = 0u32;
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if !path.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let source_html = path.join("source.html");
|
||||
if !source_html.is_file() {
|
||||
continue;
|
||||
}
|
||||
let dir_name = path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("unknown");
|
||||
let default_url = format!("https://example.com/test-pages/{}/", dir_name);
|
||||
|
||||
let metadata: PageMetadata = path
|
||||
.join("metadata.json")
|
||||
.is_file()
|
||||
.then(|| {
|
||||
let raw = std::fs::read_to_string(path.join("metadata.json"))
|
||||
.expect("read metadata.json");
|
||||
serde_json::from_str(&raw).expect("invalid metadata.json")
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
let url = metadata.url.as_deref().unwrap_or(&default_url).to_string();
|
||||
|
||||
let html = std::fs::read_to_string(&source_html).expect("read source.html");
|
||||
let markdown = ironclaw::tools::builtin::convert_html_to_markdown(&html, &url)
|
||||
.expect("convert_html_to_markdown failed");
|
||||
|
||||
let expected_md_path = path.join("expected.md");
|
||||
let should_check_expected =
|
||||
expected_md_path.is_file() && metadata.check_expected.unwrap_or(true);
|
||||
|
||||
if should_check_expected {
|
||||
let expected = std::fs::read_to_string(&expected_md_path).expect("read expected.md");
|
||||
let norm_actual = normalize_smart_punctuation(&normalize(&markdown));
|
||||
let norm_expected = normalize_smart_punctuation(&normalize(&expected));
|
||||
assert_eq!(
|
||||
norm_actual, norm_expected,
|
||||
"markdown mismatch for {}:\n--- actual ---\n{}\n--- expected ---\n{}",
|
||||
dir_name, norm_actual, norm_expected
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(ref contains) = metadata.contains {
|
||||
let normalized_md = normalize_smart_punctuation(&markdown);
|
||||
for s in contains {
|
||||
assert!(
|
||||
normalized_md.contains(&normalize_smart_punctuation(s)),
|
||||
"{}: markdown missing expected content: {:?}",
|
||||
dir_name,
|
||||
s
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if std::env::var("HTML_TO_MD_VERBOSE").is_ok() {
|
||||
println!("--- {} ---\n{}\n", dir_name, markdown);
|
||||
}
|
||||
converted += 1;
|
||||
}
|
||||
|
||||
assert!(
|
||||
converted > 0,
|
||||
"No test pages found (no directories with source.html in tests/test-pages/)"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
## The U.S. has long been heralded as a land of opportunity -- a place where anyone can succeed regardless of the economic class they were born into.
|
||||
|
||||
But a new report released on Monday by [Stanford University's Center on Poverty and Inequality](http://web.stanford.edu/group/scspi-dev/cgi-bin/) calls that into question.
|
||||
|
||||
The report assessed poverty levels, income and wealth inequality, economic mobility and unemployment levels among 10 wealthy countries with social welfare programs.
|
||||
|
||||
Among its key findings: the class you're born into matters much more in the U.S. than many of the other countries.
|
||||
|
||||
As the [report states](http://web.stanford.edu/group/scspi-dev/cgi-bin/publications/state-union-report): "[T]he birth lottery matters more in the U.S. than in most well-off countries."
|
||||
|
||||
But this wasn't the only finding that suggests the U.S. isn't quite living up to its reputation as a country where everyone has an equal chance to get ahead through sheer will and hard work.
|
||||
|
||||
[Related: Rich are paying more in taxes but not as much as they used to](http://money.cnn.com/2016/01/11/news/economy/rich-taxes/index.html?iid=EL)
|
||||
|
||||
The report also suggested the U.S. might not be the "jobs machine" it thinks it is, when compared to other countries.
|
||||
|
||||
It ranked near the bottom of the pack based on the levels of unemployment among men and women of prime working age. The study determined this by taking the ratio of employed men and women between the ages of 25 and 54 compared to the total population of each country.
|
||||
|
||||
The overall rankings of the countries were as follows:
|
||||
1. Finland
|
||||
2. Norway
|
||||
3. Australia
|
||||
4. Canada
|
||||
5. Germany
|
||||
6. France
|
||||
7. United Kingdom
|
||||
8. Italy
|
||||
9. Spain
|
||||
10. United States
|
||||
|
||||
The low ranking the U.S. received was due to its extreme levels of wealth and income inequality and the ineffectiveness of its "safety net" -- social programs aimed at reducing poverty.
|
||||
|
||||
[Related: Chicago is America's most segregated city](http://money.cnn.com/2016/01/05/news/economy/chicago-segregated/index.html?iid=EL)
|
||||
|
||||
The report concluded that the American safety net was ineffective because it provides only half the financial help people need. Additionally, the levels of assistance in the U.S. are generally lower than in other countries.
|
||||
|
||||
CNNMoney (New York) First published February 1, 2016: 1:28 AM ET
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"check_expected": true,
|
||||
"contains": [
|
||||
"land of opportunity",
|
||||
"birth lottery",
|
||||
"Stanford University's Center on Poverty and Inequality",
|
||||
"poverty levels, income and wealth inequality",
|
||||
"class you're born into matters much more",
|
||||
"Finland",
|
||||
"Norway",
|
||||
"United States",
|
||||
"safety net",
|
||||
"CNNMoney"
|
||||
]
|
||||
}
|
||||
Generated
+4190
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,311 @@
|
||||
## Open Journalism Project:
|
||||
|
||||
#### *Better Student Journalism*
|
||||
|
||||
We pushed out the first version of the [Open Journalism site](http://pippinlee.github.io/open-journalism-project/) in January. Our goal is for the
|
||||
site to be a place to teach students what they should know about journalism
|
||||
on the web. It should be fun too.
|
||||
|
||||
Topics like [mapping](http://pippinlee.github.io/open-journalism-project/Mapping/), [security](http://pippinlee.github.io/open-journalism-project/Security/), command
|
||||
line tools, and [open source](http://pippinlee.github.io/open-journalism-project/Open-source/) are
|
||||
all concepts that should be made more accessible, and should be easily
|
||||
understood at a basic level by all journalists. We’re focusing on students
|
||||
because we know student journalism well, and we believe that teaching maturing
|
||||
journalists about the web will provide them with an important lens to view
|
||||
the world with. This is how we got to where we are now.
|
||||
|
||||
### Circa 2011
|
||||
|
||||
In late 2011 I sat in the design room of our university’s student newsroom
|
||||
with some of the other editors: Kate Hudson, Brent Rose, and Nicholas Maronese.
|
||||
I was working as the photo editor then—something I loved doing. I was very
|
||||
happy travelling and photographing people while listening to their stories.
|
||||
|
||||
Photography was my lucky way of experiencing the many types of people
|
||||
my generation seemed to avoid, as well as many the public spends too much
|
||||
time discussing. One of my habits as a photographer was scouring sites
|
||||
like Flickr to see how others could frame the world in ways I hadn’t previously
|
||||
considered.
|
||||
|
||||
topleftpixel.com
|
||||
|
||||
I started discovering beautiful things the [web could do with images](http://wvs.topleftpixel.com/13/02/06/timelapse-strips-homewood.htm):
|
||||
things not possible with print. Just as every generation revolts against
|
||||
walking in the previous generations shoes, I found myself questioning the
|
||||
expectations that I came up against as a photo editor. In our newsroom
|
||||
the expectations were built from an outdated information world. We were
|
||||
expected to fill old shoes.
|
||||
|
||||
So we sat in our student newsroom—not very happy with what we were doing.
|
||||
Our weekly newspaper had remained essentially unchanged for 40+ years.
|
||||
Each editorial position had the same requirement every year. The *big* change
|
||||
happened in the 80s when the paper started using colour. We’d also stumbled
|
||||
into having a website, but it was updated just once a week with the release
|
||||
of the newspaper.
|
||||
|
||||
Information had changed form, but the student newsroom hadn’t, and it
|
||||
was becoming harder to romanticize the dusty newsprint smell coming from
|
||||
the shoes we were handed down from previous generations of editors. It
|
||||
was, we were told, all part of “becoming a journalist.”
|
||||
|
||||
### We don’t know what we don’t know
|
||||
|
||||
We spent much of the rest of the school year asking “what should we be
|
||||
doing in the newsroom?”, which mainly led us to ask “how do we use the
|
||||
web to tell stories?” It was a straightforward question that led to many
|
||||
more questions about the web: something we knew little about. Out in the
|
||||
real world, traditional journalists were struggling to keep their jobs
|
||||
in a dying print world. They wore the same design of shoes that we were
|
||||
supposed to fill. Being pushed to repeat old, failing strategies and blocked
|
||||
from trying something new scared us.
|
||||
|
||||
We had questions, so we started doing some research. We talked with student
|
||||
newsrooms in Canada and the United States, and filled too many Google Doc
|
||||
files with notes. Looking at the notes now, they scream of fear. We annotated
|
||||
our notes with naive solutions, often involving scrambled and immature
|
||||
odysseys into the future of online journalism.
|
||||
|
||||
There was a lot we didn’t know. We didn’t know **how to build a mobile app**.
|
||||
We didn’t know **if we should build a mobile app**.
|
||||
We didn’t know **how to run a server**.
|
||||
We didn’t know **where to go to find a server**.
|
||||
We didn’t know **how the web worked**.
|
||||
We didn’t know **how people used the web to read news**.
|
||||
We didn’t know **what news should be on the web**.
|
||||
If news is just information, what does that even look like?
|
||||
|
||||
We asked these questions to many students at other papers to get a consensus
|
||||
of what had worked and what hadn’t. They reported similar questions and
|
||||
fears about the web but followed with “print advertising is keeping us
|
||||
afloat so we can’t abandon it”.
|
||||
|
||||
In other words, we knew that we should be building a newer pair of shoes,
|
||||
but we didn’t know what the function of the shoes should be.
|
||||
|
||||
### Common problems in student newsrooms (2011)
|
||||
|
||||
Our questioning of other student journalists in 15 student newsrooms brought
|
||||
up a few repeating issues.
|
||||
|
||||
- Lack of mentorship
|
||||
- A news process that lacked consideration of the web
|
||||
- No editor/position specific to the web
|
||||
- Little exposure to many of the cool projects being put together by professional
|
||||
newsrooms
|
||||
- Lack of diverse skills within the newsroom. Writers made up 95% of the
|
||||
personnel. Students with other skills were not sought because journalism
|
||||
was seen as “a career with words.” The other 5% were designers, designing
|
||||
words on computers, for print.
|
||||
- Not enough discussion between the business side and web efforts
|
||||
From our 2011 research
|
||||
|
||||
### Common problems in student newsrooms (2013)
|
||||
|
||||
Two years later, we went back and looked at what had changed. We talked
|
||||
to a dozen more newsrooms and weren’t surprised by our findings.
|
||||
|
||||
- Still no mentorship or link to professional newsrooms building stories
|
||||
for the web
|
||||
- Very little control of website and technology
|
||||
- The lack of exposure that student journalists have to interactive storytelling.
|
||||
While some newsrooms are in touch with what’s happening with the web and
|
||||
journalism, there still exists a huge gap between the student newsroom
|
||||
and its professional counterpart
|
||||
- No time in the current news development cycle for student newsrooms to
|
||||
experiment with the web
|
||||
- Lack of skill diversity (specifically coding, interaction design, and
|
||||
statistics)
|
||||
- Overly restricted access to student website technology. Changes are primarily
|
||||
visual rather than functional.
|
||||
- Significantly reduced print production of many papers
|
||||
- Computers aren’t set up for experimenting with software and code, and
|
||||
often locked down
|
||||
|
||||
|
||||
Newsrooms have traditionally been covered in copies of The New York Times
|
||||
or Globe and Mail. Instead newsrooms should try spend at 20 minutes each
|
||||
week going over the coolest/weirdest online storytelling in an effort to
|
||||
expose each other to what is possible. “[Hey, what has the New York Times R&D lab been up to this week?](http://nytlabs.com/)”
|
||||
|
||||
Instead of having computers that are locked down, try setting aside a
|
||||
few office computers that allow students to play and “break”, or encourage
|
||||
editors to buy their own Macbooks so they’re always able to practice with
|
||||
code and new tools on their own.
|
||||
|
||||
From all this we realized that changing a student newsroom is difficult.
|
||||
It takes patience. It requires that the business and editorial departments
|
||||
of the student newsroom be on the same (web)page. The shoes of the future
|
||||
must be different from the shoes we were given.
|
||||
|
||||
We need to rethink how long the new shoe design will be valid. It’s more
|
||||
important that we focus on the process behind making footwear than on actually
|
||||
creating a specific shoe. We shouldn’t be building a shoe to last 40 years.
|
||||
Our footwear design process will allow us to change and adapt as technology
|
||||
evolves. The media landscape will change, so having a newsroom that can
|
||||
change with it will be critical.
|
||||
|
||||
**We are building a shoe machine, not a shoe.**
|
||||
|
||||
### A train or light at the end of the tunnel: are student newsrooms changing for the better?
|
||||
|
||||
In our 2013 research we found that almost 50% of student newsrooms had
|
||||
created roles specifically for the web. **This sounds great, but is still problematic in its current state.**
|
||||
|
||||
**We designed many of these slides to help explain to ourselves what we were doing**
|
||||
|
||||
When a newsroom decides to create a position for the web, it’s often with
|
||||
the intent of having content flow steadily from writers onto the web. This
|
||||
is a big improvement from just uploading stories to the web whenever there
|
||||
is a print issue. *However…*
|
||||
|
||||
1. **The handoff**
|
||||
Problems arise because web editors are given roles that absolve the rest
|
||||
of the editors from thinking about the web. All editors should be involved
|
||||
in the process of story development for the web. While it’s a good idea
|
||||
to have one specific editor manage the website, contributors and editors
|
||||
should all play with and learn about the web. Instead of “can you make
|
||||
a computer do XYZ for me?”, we should be saying “can you show me how to
|
||||
make a computer do XYZ?”
|
||||
2. **Not just social media**
|
||||
A
|
||||
web editor could do much more than simply being in charge of the social
|
||||
media accounts for the student paper. Their responsibility could include
|
||||
teaching all other editors to be listening to what’s happening online.
|
||||
The web editor can take advantage of live information to change how the
|
||||
student newsroom reports news in real time.
|
||||
3. **Web (interactive) editor**
|
||||
The
|
||||
goal of having a web editor should be for someone to build and tell stories
|
||||
that take full advantage of the web as their medium. Too often the web’s
|
||||
interactivity is not considered when developing the story. The web then
|
||||
ends up as a resting place for print words.
|
||||
|
||||
|
||||
Editors at newsrooms are still figuring out how to convince writers of
|
||||
the benefit to having their content online. There’s still a stronger draw
|
||||
to writers seeing their name in print than on the web. Showing writers
|
||||
that their stories can be told in new ways to larger audiences is a convincing
|
||||
argument that the web is a starting point for telling a story, not its
|
||||
graveyard.
|
||||
|
||||
When everyone in the newsroom approaches their website with the intention
|
||||
of using it to explore the web as a medium, they all start to ask “what
|
||||
is possible?” and “what can be done?” You can’t expect students to think
|
||||
in terms of the web if it’s treated as a place for print words to hang
|
||||
out on a web page.
|
||||
|
||||
We’re OK with this problem, if we see newsrooms continue to take small
|
||||
steps towards having all their editors involved in the stories for the
|
||||
web.
|
||||
|
||||
The current Open Journalism site was a few years in the making. This was
|
||||
an original launch page we use in 2012
|
||||
|
||||
### What we know
|
||||
|
||||
- **New process**
|
||||
Our rough research has told us newsrooms need to be reorganized. This
|
||||
includes every part of the newsroom’s workflow: from where a story and
|
||||
its information comes from, to thinking of every word, pixel, and interaction
|
||||
the reader will have with your stories. If I was a photo editor that wanted
|
||||
to re-think my process with digital tools in mind, I’d start by asking
|
||||
“how are photo assignments processed and sent out?”, “how do we receive
|
||||
images?”, “what formats do images need to be exported in?”, “what type
|
||||
of screens will the images be viewed on?”, and “how are the designers getting
|
||||
these images?” Making a student newsroom digital isn’t about producing
|
||||
“digital manifestos”, it’s about being curious enough that you’ll want
|
||||
to to continue experimenting with your process until you’ve found one that
|
||||
fits your newsroom’s needs.
|
||||
- **More (remote) mentorship**
|
||||
Lack of mentorship is still a big problem. [Google’s fellowship program](http://www.google.com/get/journalismfellowship/) is great. The fact that it
|
||||
only caters to United States students isn’t. There are only a handful of
|
||||
internships in Canada where students interested in journalism can get experience
|
||||
writing code and building interactive stories. We’re OK with this for now,
|
||||
as we expect internships and mentorship over the next 5 years between professional
|
||||
newsrooms and student newsrooms will only increase. It’s worth noting that
|
||||
some of that mentorship will likely be done remotely.
|
||||
- **Changing a newsroom culture**
|
||||
Skill diversity needs to change. We encourage every student newsroom we
|
||||
talk to, to start building a partnership with their school’s Computer Science
|
||||
department. It will take some work, but you’ll find there are many CS undergrads
|
||||
that love playing with web technologies, and using data to tell stories.
|
||||
Changing who is in the newsroom should be one of the first steps newsrooms
|
||||
take to changing how they tell stories. The same goes with getting designers
|
||||
who understand the wonderful interactive elements of the web and students
|
||||
who love statistics and exploring data. Getting students who are amazing
|
||||
at design, data, code, words, and images into one room is one of the coolest
|
||||
experience I’ve had. Everyone benefits from a more diverse newsroom.
|
||||
|
||||
|
||||
### What we don’t know
|
||||
|
||||
- **Sharing curiosity for the web**
|
||||
We don’t know how to best teach students about the web. It’s not efficient
|
||||
for us to teach coding classes. We do go into newsrooms and get them running
|
||||
their first code exercises, but if someone wants to learn to program, we
|
||||
can only provide the initial push and curiosity. We will be trying out
|
||||
“labs” with a few schools next school year to hopefully get a better idea
|
||||
of how to teach students about the web.
|
||||
- **Business**
|
||||
We don’t know how to convince the business side of student papers that
|
||||
they should invest in the web. At the very least we’re able to explain
|
||||
that having students graduate with their current skill set is painful in
|
||||
the current job market.
|
||||
- **The future**
|
||||
We don’t know what journalism or the web will be like in 10 years, but
|
||||
we can start encouraging students to keep an open mind about the skills
|
||||
they’ll need. We’re less interested in preparing students for the current
|
||||
newsroom climate, than we are in teaching students to have the ability
|
||||
to learn new tools quickly as they come and go.
|
||||
|
||||
Another slide from 2012 website
|
||||
|
||||
|
||||
|
||||
### What we’re trying to share with others
|
||||
|
||||
- **A concise guide to building stories for the web**
|
||||
There are too many options to get started. We hope to provide an opinionated
|
||||
guide that follows both our experiences, research, and observations from
|
||||
trying to teach our peers.
|
||||
|
||||
|
||||
Student newsrooms don’t have investors to please. Student newsrooms can
|
||||
change their website every week if they want to try a new design or interaction.
|
||||
As long as students start treating the web as a different medium, and start
|
||||
building stories around that idea, then we’ll know we’re moving forward.
|
||||
|
||||
### A note to professional news orgs
|
||||
|
||||
We’re also asking professional newsrooms to be more open about their process
|
||||
of developing stories for the web. You play a big part in this. This means
|
||||
writing about it, and sharing code. We need to start building a bridge
|
||||
between student journalism and professional newsrooms.
|
||||
|
||||
2012
|
||||
|
||||
### This is a start
|
||||
|
||||
We going to continue slowly growing the content on [Open Journalism](http://pippinlee.github.io/open-journalism-project/). We still consider this the beta version,
|
||||
but expect to polish it, and beef up the content for a real launch at the
|
||||
beginning of the summer.
|
||||
|
||||
We expect to have more original tutorials as well as the beginnings of
|
||||
what a curriculum may look like that a student newsroom can adopt to start
|
||||
guiding their transition to become a web first newsroom. We’re also going
|
||||
to be working with the [Queen’s Journal](http://queensjournal.ca/) and [The Ubyssey](http://ubyssey.ca/)next school year to better understand how to make the student
|
||||
newsroom a place for experimenting with telling stories on the web. If
|
||||
this sound like a good idea in your newsroom, we’re still looking to add
|
||||
1 more school.
|
||||
|
||||
We’re trying out some new shoes. And while they’re not self-lacing, and
|
||||
smell a bit different, we feel lacing up a new pair of kicks can change
|
||||
a lot.
|
||||
|
||||
**Let’s talk. Let’s listen.**
|
||||
|
||||
**We’re still in the early stages of what this project will look like, so if you want to help or have thoughts, let’s talk.**
|
||||
|
||||
[**[email protected]**](mailto:[email protected])
|
||||
|
||||
*This isn’t supposed to be a****manifesto™©*** *we just think it’s pretty cool to share what we’ve learned so far, and hope you’ll do the same. We’re all in this together.*
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"check_expected": true,
|
||||
"contains": [
|
||||
"Open Journalism Project",
|
||||
"Better Student Journalism",
|
||||
"Circa 2011",
|
||||
"Kate Hudson, Brent Rose, and Nicholas Maronese",
|
||||
"Flickr",
|
||||
"topleftpixel",
|
||||
"We don't know what we don't know",
|
||||
"shoe machine",
|
||||
"Queen's Journal",
|
||||
"Let's talk. Let's listen.",
|
||||
"Common problems in student newsrooms"
|
||||
]
|
||||
}
|
||||
Generated
+705
File diff suppressed because one or more lines are too long
@@ -0,0 +1,46 @@
|
||||
Virtual reality has officially reached the consoles. And it’s pretty good! [Sony’s PlayStation VR](http://finance.yahoo.com/news/review-playstation-vr-is-comfortable-and-affordable-but-lacks-must-have-games-165053851.html) is extremely comfortable and reasonably priced, and while it’s lacking killer apps, it’s loaded with lots of interesting ones.
|
||||
|
||||
But which ones should you buy? I’ve played just about every launch game, and while some are worth your time, others you might want to skip. To help you decide what’s what, I’ve put together this list of the eight PSVR games worth considering.
|
||||
|
||||
### [“Rez Infinite” ($30)](https://www.playstation.com/en-us/games/rez-infinite-ps4/)
|
||||
|
||||
Beloved cult hit “Rez” gets the VR treatment to help launch the PSVR, and the results are terrific. It includes a fully remastered take on the original “Rez” – you zoom through a Matrix-like computer system, shooting down enemies to the steady beat of thumping electronica – but the VR setting makes it incredibly immersive. It gets better the more you play it, too; unlock the amazing Area X mode and you’ll find yourself flying, shooting and bobbing your head to some of the trippiest visuals yet seen in VR.
|
||||
|
||||
### [“Thumper” ($20)](https://www.playstation.com/en-us/games/thumper-ps4/)
|
||||
|
||||
What would happen if Tron, the board game Simon, a Clown beetle, Cthulhu and a noise band met in VR? Chaos, for sure, and also “Thumper.” Called a “violent rhythm game” by its creators, “Thumper” is, well, a violent rhythm game that’s also a gorgeous, unsettling and totally captivating assault on the senses. With simple controls and a straightforward premise – click the X button and the analog stick in time with the music as you barrel down a neon highway — it’s one of the rare games that works equally well both in and out of VR. But since you have PSVR, play it there. It’s marvelous.
|
||||
|
||||
### [“Until Dawn: Rush of Blood” ($20)](https://www.playstation.com/en-us/games/until-dawn-rush-of-blood-ps4/)
|
||||
|
||||
Cheeky horror game “Until Dawn” was a breakout hit for the PS4 last year, channeling the classic “dumb teens in the woods” horror trope into an effective interactive drama. Well, forget all that if you fire up “Rush of Blood,” because this one sticks you front and center on a rollercoaster ride from Hell. Literally. You ride through a dimly-lit carnival of terror, dual-wielding pistols as you take down targets, hideous pig monsters and, naturally, maniac clowns. Be warned: If the bad guys don’t get you, the jump scares will.
|
||||
|
||||
### [“Headmaster” ($20)](https://www.playstation.com/en-us/games/headmaster-ps4/)
|
||||
|
||||
Soccer meets “Portal” in the weird (and weirdly fun) “Headmaster,” a game about heading soccer balls into nets, targets and a variety of other things while stuck in some diabolical training facility. While at first it seems a little basic, increasingly challenging shots and a consistently entertaining narrative keep it from running off the pitch. Funny, ridiculous and as easy as literally moving your head back and forth, it’s a pleasant PSVR surprise.
|
||||
|
||||
### [“RIGS: Mechanized Combat League” ($50)](https://www.playstation.com/en-us/games/rigs-mechanized-combat-league-ps4/)
|
||||
|
||||
Giant mechs + sports? That’s the gist of this robotic blast-a-thon, which pits two teams of three against one another in gorgeous, explosive and downright fun VR combat. At its best, “RIGS” marries the thrill of fast-paced competitive shooters with the insanity of piloting a giant mech in VR. It can, however, be one of the barfier PSVR games. So pack your Dramamine, you’re going to have to ease yourself into this one.
|
||||
|
||||
### [“Batman Arkham VR” ($20)](https://www.playstation.com/en-us/games/batman-arkham-vr-ps4/)
|
||||
|
||||
“I’m Batman,” you will say. And you’ll actually be right this time, because you are Batman in this detective yarn, and you know this because you actually grab the famous cowl and mask, stick it on your head, and stare into the mirrored reflection of Rocksteady Games’ impressive Dark Knight character model. It lacks the action of its fellow “Arkham” games and runs disappointingly short, but it’s a high-quality experience that really shows off how powerfully immersive VR can be.
|
||||
|
||||
### [“Job Simulator” ($30)](https://www.playstation.com/en-us/games/job-simulator-the-2050-archives-ps4/)
|
||||
|
||||
There are a number of good VR ports in the PSVR launch lineup, but the HTC Vive launch game “Job Simulator” might be the best. Your task? Lots of tasks, actually, from cooking food to fixing cars to working in an office, all for robots, because did I mention you were in the future? Infinitely charming and surprisingly challenging, it’s a great showpiece for VR.
|
||||
|
||||
### [“Eve Valkyrie” ($60)](https://www.playstation.com/en-us/games/eve-valkyrie-ps4/)
|
||||
|
||||
Already a hit on the Oculus Rift, this space dogfighting game was one of the first to really show off how VR can turn a traditional game experience into something special. It’s pricey and not quite as hi-res as the Rift version, but “Eve Valkyrie” does an admirable job filling the void left since “Battlestar Galactica” ended. Too bad there aren’t any Cylons in it (or are there?)
|
||||
|
||||
***More games news:***
|
||||
|
||||
- [‘Skylanders Imaginators’ will let you create and 3D print your own action figures](https://www.yahoo.com/tech/skylanders-imaginators-will-let-you-create-and-3d-print-your-own-action-figure-143838550.html)
|
||||
- [Review: High-flying ‘NBA 2K17’ has a career year](https://www.yahoo.com/tech/review-high-flying-nba-2k17-has-a-career-year-184135248.html)
|
||||
- [Review: Race at your own speed in big, beautiful ‘Forza Horizon 3’](https://www.yahoo.com/tech/review-race-at-your-own-speed-in-big-beautiful-forza-horizon-3-195337170.html)
|
||||
- [Sony’s PlayStation 4 Pro shows promise, potential and plenty of pretty lighting](https://www.yahoo.com/tech/sonys-playstation-4-pro-shows-promise-potential-161304037.html)
|
||||
- [Review: ‘Madden NFL 17’ runs hard, plays it safe](https://www.yahoo.com/tech/review-madden-nfl-17-runs-000000394.html)
|
||||
|
||||
|
||||
*Ben Silverman is on Twitter at*[*ben_silverman*](https://twitter.com/ben_silverman)*.*
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"check_expected": true,
|
||||
"contains": [
|
||||
"Virtual reality has officially reached",
|
||||
"RIGS",
|
||||
"Dramamine",
|
||||
"Rez Infinite",
|
||||
"Thumper",
|
||||
"Until Dawn",
|
||||
"Headmaster",
|
||||
"Batman Arkham VR",
|
||||
"Job Simulator",
|
||||
"Eve Valkyrie",
|
||||
"Battlestar Galactica",
|
||||
"Ben Silverman",
|
||||
"eight PSVR games worth considering",
|
||||
"More games news"
|
||||
]
|
||||
}
|
||||
Generated
+14670
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user