# CiteLink — https://www.citelink.org
#
# Everything public is open to crawlers. The disallows below are not there to
# hide anything: they are query-string surfaces where a crawler would otherwise
# generate an unbounded number of near-identical URLs from filter combinations.

User-agent: *
Allow: /

# Private surfaces
Disallow: /panel.html
Disallow: /auth.html

# Faceted search: every filter combination is a distinct URL and there are
# millions of them. The canonical, unfiltered pages are indexed instead.
Disallow: /articles.html?*
Disallow: /journals.html?*
Disallow: /authors.html?*
Disallow: /organizations.html?*

Sitemap: https://www.citelink.org/sitemap.xml