# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
User-agent: *
Disallow: /*;*
Disallow: /job/go_to_apply_url/

# Allow everything else
Allow: /

# Prevent crawlers from indexing overly deep segmented URLs.
# This pattern matches THREE segments and up, not four as the comment here long claimed. The two
# Allow rules below exist because of that: robots.txt resolves by LONGEST match, so `Allow:
# /resources/` (11 characters) beats `Disallow: /*/*/*` (6) and the editorial pages get through.
Disallow: /*/*/*

# Editorial and the job-posting funnel sit three segments deep and must stay crawlable. Without
# these two lines they were submitted in the sitemap and disallowed at the same time, which is the
# worst of both: on 2026-08-24 the URL Inspection API answered "URL is unknown to Google" for
# /resources/design-careers/323-ui-designer. Never crawled once since publication.
Allow: /resources/
Allow: /job/post/

# /blog paginates with ?page=N, which `Disallow: /*?*` blocks, so 49 of 59 posts have no short
# crawlable path: each article links only to its two date neighbours, and article 1 sits some
# 58 hops from /blog. 13 of the 61 blog URLs inspected on 2026-08-24 had never been fetched.
# `Allow: /blog` would be the obvious rule and it is wrong: it also unblocks
# /blogging-newsletters?token=... . Anchor on the pagination parameter instead.
Allow: /blog?page=
Disallow: /blog?page=*&

# Same for the companies listings, which paginate with ?page=N and were all past page 1 behind
# `Disallow: /*?*` (2026-10). Anchored on the parameter, so their ?query= and ?sort= variants stay out.
Allow: /creative-companies-hiring?page=
Allow: /creative-companies-hiring/*?page=
# Prefix matching would also let in ?page=2&query=... and ?page=2&sort=... (the order the "load more"
# button writes in the address bar). Each Disallow below is longer than its Allow, so it wins.
Disallow: /creative-companies-hiring?page=*&
Disallow: /creative-companies-hiring/*?page=*&
# And /latest-jobs, whose pages 2+ list the rest of the last 30 days' jobs: a short crawl path to
# fresh postings.
Allow: /latest-jobs?page=
Disallow: /latest-jobs?page=*&

# Optional: disallow search pages or other query-heavy URLs
Disallow: /*?*

# Sitemaps
Sitemap: https://www.woodyjobs.com/sitemap.xml
Sitemap: https://www.woodyjobs.com/sitemap_general.xml
Sitemap: https://www.woodyjobs.com/sitemap_available_jobs.xml
Sitemap: https://www.woodyjobs.com/sitemap_hubs.xml
Sitemap: https://www.woodyjobs.com/sitemap_jobs_by_countries.xml
Sitemap: https://www.woodyjobs.com/sitemap_jobs_by_cities.xml
Sitemap: https://www.woodyjobs.com/sitemap_jobs_by_categories.xml
Sitemap: https://www.woodyjobs.com/sitemap_jobs_by_industries.xml
Sitemap: https://www.woodyjobs.com/sitemap_companies.xml
Sitemap: https://www.woodyjobs.com/sitemap_companies_by_industries.xml

User-agent: ClaudeBot
Disallow: /*/*/*
Crawl-delay: 5