aa96ecb9da291fbb49c66ec5871f206a *DESCRIPTION
5108db10e8669a00d81ea056076ec0a2 *LICENSE
d6631a9c6caf2369fd72772d6ae490ae *NAMESPACE
71c58b838873e9bb8d4536169b8040a3 *NEWS.md
d5436daf68a3991a73f8643b05fd5b60 *R/as_list.R
eb1449724d7d06fa8c193e4318c289a2 *R/fix_url.R
b18c2fb2a7e2e818ccfccdc0a0516354 *R/get_robotstxt.R
15f4b8a1cf3c61726f8d6360f9c2aeec *R/get_robotstxt_http_get.R
3de8fd8cd1f33204a0a48f8ecb826c73 *R/get_robotstxts.R
7472469d031d4cc751f68cccf5dcc424 *R/guess_domain.R
30f11baf27a755ce755cd92b4c187ffb *R/http_domain_changed.R
3e1561398ba6d10fcc450b2360ddd197 *R/http_subdomain_changed.R
76b4c93d4638ffd0279cd9583f314d71 *R/http_was_redirected.R
2f273ff5898ef1fde53c720e8380226e *R/is_suspect_robotstxt.R
eaea9c52d783a066f56173c26bdc0350 *R/is_valid_robotstxt.R
e5838fbfe828ba958729a2ff69c5b7d0 *R/list_merge.R
d95a8cf48d7d22ca02a8df6e2e5660c0 *R/null_to_default.R
e053344fd0cb77fe853d56b25525b0ca *R/parse_robotstxt.R
df42b98d5d1adedf3c8e7463d8979896 *R/parse_url.R
70d3712ba548fe22b07d69c81c8cb443 *R/paths_allowed.R
18e347fadbeaea7ebac55e371822ab98 *R/paths_allowed_worker_spiderbar.R
e002670688bb09fd0d464f674968fc65 *R/pipe.R
3df4af40a6b8b0073c2c913c2aea1afd *R/print_robotstxt.R
9f03da4cb9869e07fa23c294bf956dff *R/print_robotstxt_text.R
ce384c1325b4c9499f63aea978ac9da6 *R/remove_domain.R
41bbd61f242e6d18ee3c1d0c98a5fbdf *R/request_handler_handler.R
687ed2308d59083f97e20c01f21071ab *R/robotstxt.R
7d34b0e8417b244318a5e39c5ae010ec *R/rt_cache.R
716845e77f07f6f8f91132581ab79845 *R/rt_get_comments.R
21773ef62401c48c52527453ae1f4d90 *R/rt_get_fields.R
ec72b13ccc25816f202e1fd05a25a4fc *R/rt_get_fields_worker.R
d495b4af455965bf0f4e2cf68305bdd9 *R/rt_get_useragent.R
883b06f0ccccf1d7ec084d932c84ea0f *R/rt_request_handler.R
33690fcef9153e316669f8b1e6d81547 *R/rt_request_handler_defaults.R
dd0f9f184bba5559b9ebf606b66a57e7 *R/sanitize_path.R
b1f74ca0815a34b1ef3718e05e2ab9ea *R/tools.R
03be8e32415c73ce8f48fcc017b2d08e *README.md
783e82fb012cb7a4a6e9000fafa8df9d *build/vignette.rds
b40fbd096716a3f173644053c37e8438 *inst/doc/using_robotstxt.R
04c38b00d15b1e5a06727abe7bfff1eb *inst/doc/using_robotstxt.Rmd
f863d5771f496534ec7f6dbd945a7f91 *inst/doc/using_robotstxt.html
4600b105324e0e285ad6384f5f01968d *inst/http_requests/http_404.rds
48598696c69b2826c3e9d919c4377369 *inst/http_requests/http_client_error.rds
bb9f78439e4ded437aaac70a3967e79b *inst/http_requests/http_domain_change.rds
513a4bc54cbd540cfd4f1057a46d2d7e *inst/http_requests/http_html_content.rds
e8295be102459c31b63b9faee4e4c519 *inst/http_requests/http_ok_1.rds
cda4365dc8a9e29eb2623c9428be8047 *inst/http_requests/http_ok_2.rds
1dc61ada1a0a11297d653443e75b5b10 *inst/http_requests/http_ok_3.rds
44f05e9ab8d7dae5b744a5f60fcca5cc *inst/http_requests/http_ok_4.rds
09019d9676716e69aa98d8a8b02be6b0 *inst/http_requests/http_redirect_www.rds
b95430e784c92a5ef59af45e523ace00 *inst/http_requests/http_server_error.rds
4d8921c3dc225aa07cd9b3f66cde3aff *inst/robotstxts/allow_single_bot.txt
e2b27c9301b4f24c3b2db3f8886b2dc1 *inst/robotstxts/crawl_delay.txt
8cb7a8b11cc42f1f77aa0670c43071a3 *inst/robotstxts/disallow_all_for_BadBot.txt
d35cb09d5662e686347b00b959c1550f *inst/robotstxts/disallow_all_for_all.txt
542398cc4e150fd288cea549bdca16a4 *inst/robotstxts/disallow_some_for_all.txt
9ca8636f08a496a82ed0a64694d005c8 *inst/robotstxts/disallow_two_at_once.txt
7215ee9c7d9dc229d2921a40e899ec5f *inst/robotstxts/empty.txt
d034a6e564902694ac6b372940786d80 *inst/robotstxts/host.txt
105d349d6d2c34ca1a99aa50b6a79ae4 *inst/robotstxts/rbloggers.txt
95d8d7433b4cd37b5630cf68500127f0 *inst/robotstxts/robots_amazon.txt
a09430767142d30f771f9726196048e1 *inst/robotstxts/robots_bundestag.txt
6365a388da3f5b8ecafabf97be15127a *inst/robotstxts/robots_cdc.txt
6365a388da3f5b8ecafabf97be15127a *inst/robotstxts/robots_cdc2.txt
1cab7e612b824c0222d02760db298f06 *inst/robotstxts/robots_commented_token.txt
3b1a4b952779358ff7d6015b0dee0e68 *inst/robotstxts/robots_facebook.txt
c51eaab2c346b3a156c72097e52b10d9 *inst/robotstxts/robots_facebook_unsupported.txt
62dbaccb94d6ab8aa5e178339efbe127 *inst/robotstxts/robots_google.txt
658622dc6bda3645b94e37fa1ea38295 *inst/robotstxts/robots_new_york_times.txt
48a1701be03cf9590a8ef623c5f2e8e6 *inst/robotstxts/robots_pmeissner.txt
67dfdd8b9acd9363417c145388315dab *inst/robotstxts/robots_spiegel.txt
6ed07c3b696c3b4d47e1b3b4a42f5c7f *inst/robotstxts/robots_wikipedia.txt
b03c30fd553878fce698924ee1eca06e *inst/robotstxts/robots_wikipedia_20170706.txt
4637c3108489f0f0e5e283ef16884baf *inst/robotstxts/robots_yahoo.txt
6b8539213a6199601ee9d81c70a6ac31 *inst/robotstxts/selfhtml_Example.txt
2b9b53034510e3b66af8968145fe0a4f *inst/robotstxts/testing_comments.txt
67852bf43216da4632fbae410103e712 *inst/urls.txt
831093032e270a2f516380a89096b0a1 *man/as.list.robotstxt_text.Rd
53a5de5ee843b1b4902e4fed3d9690bd *man/fix_url.Rd
be9ee5d0eb602d68600433d1f7f963b6 *man/get_robotstxt.Rd
f0b458d14a318962e07e23b536a80f62 *man/get_robotstxt_http_get.Rd
6ab809fc41dc93147324e23fb3580a4d *man/get_robotstxts.Rd
d3f254e162a37fda4c32cd7ce78b7f11 *man/guess_domain.Rd
a2aba0b74a34d7dc5cdd132f29b4968f *man/http_domain_changed.Rd
cfded2d7606869affa85fec985320e86 *man/http_subdomain_changed.Rd
650233d588f6f4b2022cf3faa63a50c8 *man/http_was_redirected.Rd
1e1a6731f4d9f6473011ad37b898f8bb *man/is_suspect_robotstxt.Rd
0c27d302f335d8b6c1af032f4fab4ab2 *man/is_valid_robotstxt.Rd
e95b0adb4e98e040776320413d1d0cb6 *man/list_merge.Rd
714978c33b94323e62aeb3e279a98f20 *man/named_list.Rd
63a9829d9ed5af511b9facdf5831b00b *man/null_to_defeault.Rd
966b8100baa85950dacb4e7fb90b9381 *man/parse_robotstxt.Rd
930edad24bbac466e8a5bd0b2a9dd81a *man/parse_url.Rd
6ca84d3cd4bb28e805d926642d603d1a *man/paths_allowed.Rd
ed8a624cfd48e11ae47815b252bd0c6e *man/paths_allowed_worker_spiderbar.Rd
cb7c8c804b15c022249d72fd4d10abe9 *man/pipe.Rd
5efbce30d75786dbd24ee148c2cbfadf *man/print.robotstxt.Rd
a290662dbf85cd54a676074f719aa830 *man/print.robotstxt_text.Rd
e5f60861c1292e29d6567d41093506a3 *man/remove_domain.Rd
3946561a85a72c72f37c68633dceef65 *man/request_handler_handler.Rd
f12b9529edfe7a1e431f4fc897842427 *man/robotstxt.Rd
c797a59894377a91be74ab5903db3d2d *man/rt_cache.Rd
47dc136b56c04c338be8a62943ebe7f8 *man/rt_get_comments.Rd
9eee10f3ea9bdcefb10cf1a4fd41c3b3 *man/rt_get_fields.Rd
2f946d9ef928f260d6f5c3600476d7b8 *man/rt_get_fields_worker.Rd
e85d1e0c1dc30046ae5880ab45810166 *man/rt_get_rtxt.Rd
0e08232c6db42344af5dcf49648e7c6c *man/rt_get_useragent.Rd
aa83e831581c502e3d07a1dbc481ce11 *man/rt_list_rtxt.Rd
388b08e26c09455afb955090af2bab1c *man/rt_request_handler.Rd
6ec25121789e8273a809bd4f97f13ce3 *man/sanitize_path.Rd
480e44e1c814faf1886f057b8a375590 *tests/testthat.R
eb607f08a622b22e2f28d1b446ca4ff0 *tests/testthat/test_attribute_handling.R
f898f8eb85364f39f391b7d764ae7f51 *tests/testthat/test_get_robotstxt.R
d2a11d123e0c6a28b1b025ed4f74d8d4 *tests/testthat/test_http_event_handling.R
864e5273d6cc056546f01778335f3e38 *tests/testthat/test_issue50.R
34497065f651a00bb73c49fea23bfd7d *tests/testthat/test_parser.R
7082e9f8c01fe1e5f734020b1a096740 *tests/testthat/test_path_examples_from_rfc.R
a659fb2cedac0c452737c4bf9a07b771 *tests/testthat/test_paths_allowed.R
7bbf52d91aa28203eb59f2977e37f5fd *tests/testthat/test_robotstxt.R
746b9e51cf6a485dd90a685a93607b67 *tests/testthat/test_tools.R
f8b3499a876f34f4164a0cc4fb47092f *vignettes/style.css
04c38b00d15b1e5a06727abe7bfff1eb *vignettes/using_robotstxt.Rmd
