Live HTTP Target Headers
{"date":"Sun, 08 Feb 2026 12:19:02 GMT","content-type":"text/html; charset=utf-8","server":"cloudflare","x-origin-cache":"HIT","last-modified":"Thu, 29 Jan 2026 02:44:51 GMT","access-control-allow-origin":"*","expires":"Sun, 08 Feb 2026 11:37:37 GMT","cache-control":"max-age=600","report-to":"{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=9UVIEsLKmwYzJKyTEGU9eJbfoa9mKOohtPjJZMSH4k7vh4PZiBoaV5RzIE363lD9TrsVvY2eiFJAVNYhKVsAEmxGznkWG0t6e8GH"}]}","x-proxy-cache":"MISS","x-github-request-id":"F81C:1D2EA5:219F3B9:220C94D:69887328","nel":"{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}","age":"0","via":"1.1 varnish","x-served-by":"cache-lin1730074-LIN","x-cache":"HIT","x-cache-hits":"0","x-timer":"S1770553143.625622,VS0,VE115","vary":"Accept-Encoding","x-fastly-request-id":"2d4793e18da6e401b01c135a433a4bfea4486c4a","cf-cache-status":"DYNAMIC","content-encoding":"br","cf-ray":"9cab12b55832ed68-MXP","alt-svc":"h3=":443"; ma=86400"} Live HTTP File Headers (LLMS)
{"date":"Sun, 08 Feb 2026 12:19:02 GMT","content-type":"text/plain; charset=utf-8","content-length":"370","server":"cloudflare","last-modified":"Thu, 29 Jan 2026 02:44:51 GMT","access-control-allow-origin":"*","etag":"W/"697ac9a3-288"","expires":"Sun, 08 Feb 2026 12:29:02 GMT","cache-control":"max-age=600","content-encoding":"gzip","x-proxy-cache":"MISS","x-github-request-id":"0946:1C9918:21F232E:2262083:69887F35","accept-ranges":"bytes","age":"0","via":"1.1 varnish","x-served-by":"cache-lin1730066-LIN","x-cache":"MISS","x-cache-hits":"0","x-timer":"S1770553142.320416,VS0,VE115","vary":"Accept-Encoding","x-fastly-request-id":"3492ecb1ea1216fb90b8d2e6ac68f5c07849e509","cf-cache-status":"DYNAMIC","nel":"{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}","report-to":"{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=kK21pwUrX9uEmomkoa4sDznGWM5vHj5GqpV9WbczOwwnls%2BVH4eVA41sXZtfe8VY37M7pFczP8lZ51egovDpDKC3XsSxI2W6Kxrl"}]}","cf-ray":"9cab12b35e08ee7c-MXP","alt-svc":"h3=":443"; ma=86400"} Raw AI Engine Response (JSON)
{
"taxonomy": {
"country": null,
"categories": [
"AI & Machine Learning",
"Developer Tools",
"Documentation",
"Open Source"
],
"tags": [
"Agentic Web",
"Machine-Legibility",
"Markdown-Optimized",
"RAG-Ready",
"Token Efficiency",
"Generative Engine Optimization (GEO)"
],
"topics": [
"LLMs.txt Spec",
"LLM Documentation",
"Markdown Publishing"
],
"category_suggestions": [],
"tag_suggestions": []
},
"brand": {
"name": "llms-txt",
"description": "An open proposal and documentation hub for standardizing a /llms.txt file that provides LLM-friendly, markdown-based guidance and curated links for inference-time use.",
"type": "open source project",
"industry": "AI developer documentation / web standards",
"business_model": "open specification + open-source tooling (docs, libraries, plugins)",
"target_audience": "Website owners, documentation teams, and developers building LLM/agent integrations that need curated, machine-readable site context.",
"tone_voice": [
"technical",
"instructional",
"community-oriented",
"proposal/spec-focused"
],
"contact_email": null,
"uniqueness": "Defines a simple, Markdown-structured convention at a standard path (/llms.txt) plus optional .md page variants to make websites easier for LLMs/agents to consume.",
"market_position": "Reference proposal site for the llms.txt convention, with links to implementations and integrations."
},
"languages": {
"primary": "en",
"others": [],
"localization": "global"
},
"analysis": {
"robots": {
"sentiment": "neutral",
"observation": "robots.txt only declares a sitemap location and does not specify any allow/disallow rules.",
"tip": "If you want to express crawler/AI access preferences, add explicit User-agent rules and/or pair with a clear policy statement in llms.txt.",
"facts": [
"robots.txt contains: \"Sitemap: https://llmstxt.org/sitemap.xml\""
],
"issues": [
"No User-agent directives (Allow/Disallow) present in robots.txt."
]
},
"llms": {
"sentiment": "positive",
"observation": "llms.txt is present at /llms.txt, uses Markdown, includes a short blockquote summary, and provides a curated \"Docs\" section with three annotated links (proposal, Python library docs, and an example).",
"tip": "Consider adding an explicit license/usage note (training vs inference, attribution) and additional sections (e.g., \"Optional\") if you want to guide context truncation.",
"facts": [
"llms.txt H1: \"# llms.txt\"",
"Blockquote summary describes the purpose: adding /llms.txt to provide LLM-friendly background and links",
"Includes section \"## Docs\" with links to https://llmstxt.org/index.md, https://llmstxt.org/intro.html.md, and https://llmstxt.org/ed-commonmark.md"
],
"issues": [
"No explicit license or usage permissions stated in llms.txt.",
"No explicit policy guidance on training vs inference beyond general description on the homepage (not in llms.txt)."
]
},
"jsonld": {
"sentiment": "negative",
"observation": "No JSON-LD structured data was provided in the scrape (json_ld is empty).",
"tip": "Add basic Schema.org markup (e.g., Organization/WebSite/Article) to improve machine understanding and attribution.",
"facts": [],
"issues": [
"Missing JSON-LD structured data in provided data."
]
},
"website": {
"sentiment": "positive",
"observation": "The homepage clearly explains the motivation, proposed format, and examples for llms.txt, and links to tools, directories, and integrations. It is strongly oriented toward machine-readable documentation workflows (Markdown, .md URL variants).",
"tip": "Add a dedicated policy page (and link it from llms.txt) clarifying permitted uses (e.g., inference-time retrieval, caching) and attribution expectations.",
"facts": [
"Homepage describes proposal to add /llms.txt and provide clean Markdown versions of pages via .md suffix",
"Mentions expansion tooling and context files (llms-ctx.txt and llms-ctx-full.txt) via llms_txt2ctx",
"Lists directories: llmstxt.site and directory.llmstxt.cloud",
"Lists integrations: llms_txt2ctx, JavaScript implementation, VitePress/Docusaurus plugins, Drupal recipe, PHP library, VS Code PagePilot extension",
"Author listed on page: Jeremy Howard; Published September 3, 2024"
],
"issues": [
"No explicit AI usage policy or licensing information found in provided fields (privacy_url/license_url empty).",
"No structured data (JSON-LD) provided."
]
},
"policy": {
"stance": "open",
"observation": "No restrictive signals were found in robots.txt (only sitemap). HTTP headers provided do not include x-robots-tag or tdm-reservation. llms.txt is explicitly intended to help LLMs use the site at inference time.",
"license": null,
"allowed": [
"Inference-time use guidance is explicitly described in the site content (llms.txt proposal)."
],
"denied": [],
"issues": [
"No explicit license or terms for reuse/training were found in llms.txt, robots.txt, or provided metadata.",
"No machine-readable TDM/AI policy headers (e.g., tdm-reservation) observed in provided headers."
]
}
},
"ai_content": {
"json_feeds": [],
"mcp_server": null,
"api_endpoints": [],
"structured_data": [],
"machine_readable": [
"/llms.txt (Markdown)",
"Markdown page variants referenced (e.g., /index.md, *.html.md)",
"Sitemap: https://llmstxt.org/sitemap.xml"
],
"data_exports": [],
"embeddings_ready": true,
"crawl_friendly": true,
"notes": "Content is largely documentation/proposal text with explicit Markdown resources and permissive CORS header (access-control-allow-origin: *)."
},
"strategic": {
"rag_quality": {
"assessment": "High suitability for RAG about the llms.txt specification and related tooling, due to curated links and Markdown-first content.",
"strengths": [
"Dedicated /llms.txt with curated documentation links",
"Markdown-optimized resources (index.md, *.html.md) reduce HTML noise",
"Clear conceptual structure (proposal, format, examples, integrations)"
],
"weaknesses": [
"No explicit license/usage terms for downstream reuse",
"No JSON-LD metadata for attribution/entity resolution"
]
},
"privacy": {
"rating": "Unclear",
"assessment": "No privacy policy URL or privacy statements were provided in the scrape; site appears informational/spec-focused.",
"signals": [
"Served via Cloudflare (server: cloudflare)",
"HTTPS URL provided"
],
"concerns": [
"privacy_url not provided; no privacy terms assessed from supplied data."
]
},
"update_cycle": {
"predicted": "Occasional",
"reasoning": "Spec/proposal content with periodic updates; headers show a recent last-modified date for the site assets.",
"signals": [
"last-modified header: Thu, 29 Jan 2026 02:44:51 GMT",
"Homepage references ongoing community input and GitHub-based discussion"
]
}
},
"insights": {
"summary": "llmstxt.org is the reference proposal site for the /llms.txt convention, providing a Markdown-based, LLM-friendly entry point and curated documentation links intended primarily for inference-time use.",
"highlights": [
"/llms.txt present and well-formed with summary + curated docs links",
"Sitemap is declared in robots.txt",
"Strong emphasis on Markdown and deterministic parsing for agents/LLMs"
],
"anomalies": [
"Robots.txt contains only a sitemap line and no crawl directives."
],
"opportunities": [
"Add explicit license/terms and AI usage guidance (training vs inference, attribution) directly in llms.txt",
"Add JSON-LD (WebSite/Organization/Article) for better entity attribution and indexing"
],
"concerns": [
"Policy ambiguity: no explicit reuse/training permissions or restrictions found in provided data.",
"No structured data provided (json_ld empty)."
]
},
"quality": {
"trust_signals": [
"Clear authorship and publish date stated on homepage (Jeremy Howard; Published September 3, 2024)",
"Consistent presence of /llms.txt and linked Markdown docs",
"Sitemap advertised in robots.txt"
],
"tech_stack": "Cloudflare + GitHub Pages/Fastly caching indicated by headers (cloudflare, via varnish, x-github-request-id, x-fastly-request-id).",
"assessment": "High-quality, machine-legible documentation source for the llms.txt specification with strong RAG readiness due to Markdown resources; lacks explicit licensing/policy metadata and structured data.",
"recommendation": "Include in directory as an authoritative documentation/spec source for llms.txt; flag missing license/policy details."
}
} 