{
 "name": "BayanatLabs-OpenData crawler policy",
 "licence": "CC0-1.0",
 "policy_version": "1.0.0",
 "updated_at": "2026-10-31",
 "operator": {
  "name": "Bayanat Labs",
  "description": "Arabic AI data company",
  "city": "Riyadh",
  "country": "Saudi Arabia",
  "website": "https://bayanatlabs.com",
  "contact": "hello@bayanatlabs.com",
  "page_en": "https://bayanatlabs.com/open/crawler",
  "page_ar": "https://bayanatlabs.com/ar/open/crawler",
  "not_affiliated_with": "other organisations and websites named Bayanat"
 },
 "user_agent": "BayanatLabs-OpenData/1.0 (+https://bayanatlabs.com/open; hello@bayanatlabs.com)",
 "product_token": "BayanatLabs-OpenData",
 "contact": "hello@bayanatlabs.com",
 "robots": {
  "spec": "RFC 9309",
  "spec_url": "https://www.rfc-editor.org/rfc/rfc9309",
  "cache_seconds": 86400,
  "max_bytes": 512000,
  "max_redirects": 5,
  "on_2xx": "parse",
  "on_401_403": "stop_host",
  "on_429": "disallow_all",
  "on_other_4xx": "allow_all",
  "on_5xx": "disallow_all",
  "on_unreachable": "disallow_all",
  "group_matching": "groups naming the product token (case-insensitive) are combined; otherwise the * groups are combined",
  "rule_matching": "longest matching Allow or Disallow pattern wins; Allow wins a tie; * and $ are supported; paths are compared after percent-encoding normalisation"
 },
 "rate_limits": {
  "scope": "per host, per worker",
  "html_seconds_per_request": 3,
  "bulk_seconds_per_request": 1,
  "max_connections_per_host": 1,
  "crawl_delay": "honoured when larger than the default interval",
  "run_minimum_can_only_slow_down": true
 },
 "backoff": {
  "statuses": [
   429,
   503
  ],
  "retry_after": "seconds or HTTP-date",
  "schedule_seconds": [
   60,
   300,
   900
  ],
  "max_retry_after_seconds": 900,
  "stop_after_consecutive_backoffs": 3
 },
 "stop_rules": [
  {
   "id": "stop.forbidden",
   "when": "HTTP 403"
  },
  {
   "id": "stop.login_required",
   "when": "HTTP 401, or a redirect to a login or sign-in page"
  },
  {
   "id": "stop.waf_challenge",
   "when": "a bot-protection challenge, such as the response header cf-mitigated: challenge"
  },
  {
   "id": "stop.captcha",
   "when": "a CAPTCHA page"
  },
  {
   "id": "stop.rate_limited",
   "when": "a fourth consecutive 429 or 503 after three backoffs"
  },
  {
   "id": "stop.retry_after_too_long",
   "when": "a Retry-After longer than max_retry_after_seconds"
  }
 ],
 "stop_scope": "the host is stopped for the rest of the run; the stop is logged; challenges and CAPTCHAs are never solved",
 "cookies": "never stored or sent back",
 "purposes": {
  "training_release": "content for an openly licensed dataset release (anyone may train on it)",
  "retrieval_release": "content for a retrieval dataset release",
  "benchmark_release": "content for a benchmark release",
  "index_only": "the caller wants only the index record (URL, title, SHA-256, annotations)",
  "documented_download": "a file that the publisher's own download page or documentation offers for bulk download; needs documentation_url"
 },
 "decisions": {
  "fetch": "may be fetched and released under its licence",
  "index_only": "may be fetched; a release holds only the URL, title, SHA-256 hash and Bayanat Labs annotations",
  "skip": "not fetched",
  "stopped": "the host is stopped; not fetched"
 },
 "signal_order": [
  {
   "id": "stop.*",
   "signal": "Stop rules",
   "effect": "stopped",
   "spec_url": null
  },
  {
   "id": "robots.unreachable",
   "signal": "robots.txt unreachable (5xx, 429, no answer, too many redirects)",
   "effect": "skip",
   "spec_url": "https://www.rfc-editor.org/rfc/rfc9309"
  },
  {
   "id": "robots.disallow_token",
   "signal": "robots.txt Disallow in a group for BayanatLabs-OpenData",
   "effect": "skip",
   "spec_url": "https://www.rfc-editor.org/rfc/rfc9309"
  },
  {
   "id": "robots.disallow_wildcard",
   "signal": "robots.txt Disallow in the * group (no group names BayanatLabs-OpenData)",
   "effect": "skip, except for documented_download",
   "spec_url": "https://www.rfc-editor.org/rfc/rfc9309"
  },
  {
   "id": "redirect.too_many",
   "signal": "more than 5 redirects for the page (the robots.max_redirects limit)",
   "effect": "skip",
   "spec_url": null
  },
  {
   "id": "ai_bot_block",
   "signal": "robots.txt Disallow in a group that names a crawler in the pinned AI crawler list, unless a group for BayanatLabs-OpenData allows the path",
   "effect": "index_only",
   "spec_url": "https://github.com/ai-robots-txt/ai.robots.txt"
  },
  {
   "id": "tdmrep.reserved",
   "signal": "TDMRep tdm-reservation: 1 in /.well-known/tdmrep.json (first matching location), the HTTP header or the HTML meta tag; any source with 1 counts",
   "effect": "index_only, unless the tdm-policy grants tdm:mine with no duty and no constraint",
   "spec_url": "https://w3c.github.io/cg-reports/tdmrep/CG-FINAL-tdmrep-20240202/"
  },
  {
   "id": "content_signal.ai_train_no",
   "signal": "Content-Signal ai-train=no in robots.txt for our group (or *)",
   "effect": "index_only for every release purpose",
   "spec_url": "https://contentsignals.org/"
  },
  {
   "id": "content_signal.ai_input_no",
   "signal": "Content-Signal ai-input=no in robots.txt for our group (or *)",
   "effect": "index_only for retrieval_release and benchmark_release",
   "spec_url": "https://contentsignals.org/"
  },
  {
   "id": "aipref.restrictive",
   "signal": "IETF aipref Content-Usage (robots.txt rule or HTTP header) with any n value; draft vocabulary",
   "effect": "index_only (feature flag aipref_restrictive)",
   "spec_url": "https://datatracker.ietf.org/wg/aipref/about/"
  },
  {
   "id": "x_robots.noai",
   "signal": "X-Robots-Tag header or robots meta tag with noai or noimageai, for all crawlers or for BayanatLabs-OpenData",
   "effect": "index_only",
   "spec_url": null
  },
  {
   "id": "rsl.license",
   "signal": "RSL License: directive in robots.txt",
   "effect": "index_only (treated as a reservation)",
   "spec_url": "https://rslstandard.org/"
  },
  {
   "id": "purpose.index_only",
   "signal": "the caller asked for the index record only",
   "effect": "index_only",
   "spec_url": null
  },
  {
   "id": "default.allow",
   "signal": "no restrictive signal",
   "effect": "fetch",
   "spec_url": null
  }
 ],
 "feature_flags": {
  "aipref_restrictive": true
 },
 "documented_download": {
  "requires": "documentation_url: the publisher page that offers the file for bulk download",
  "overrides": [
   "robots.disallow_wildcard"
  ],
  "never_overrides": [
   "robots.disallow_token",
   "robots.unreachable",
   "stop.*"
  ],
  "behaviour": "one file at a time; no discovery and no link following; the documentation URL is recorded in the signals ledger"
 },
 "ai_bot_list": {
  "source": "https://github.com/ai-robots-txt/ai.robots.txt",
  "file": "robots.json",
  "commit": "d8e8b77ca6f04ee132fd5dec0ce842b5d45e85da",
  "commit_date": "2026-10-30",
  "licence": "MIT",
  "count": 202,
  "tokens_url": "https://github.com/Bayanat-Labs/polite-fetch/blob/v0.1.0/src/bayanatlabs_polite_fetch/data/ai_bots.json",
  "sha256": "5b6c4936711807dfe963afc1346c9b8b473a5752c926dc67ea520f38fad8a478"
 },
 "verification": {
  "ip_list": false,
  "how": "Requests come from the machines that run Bayanat Labs pipelines. Bayanat Labs does not publish an IP list. Check the User-Agent string and write to hello@bayanatlabs.com to confirm a request."
 },
 "removal": {
  "how": "Email hello@bayanatlabs.com with the URL or the record id.",
  "effect": "The record is removed in the next version of that dataset."
 },
 "ledger_schema_url": "https://github.com/Bayanat-Labs/polite-fetch/blob/v0.1.0/docs/signals.schema.json",
 "library": {
  "name": "bayanatlabs-polite-fetch",
  "repository": "https://github.com/Bayanat-Labs/polite-fetch",
  "version": "0.1.0",
  "licence": "Apache-2.0"
 },
 "generated_from": "https://github.com/Bayanat-Labs/polite-fetch/blob/v0.1.0/policy/crawler-policy.yaml"
}