User-agent: * Disallow: /accounts Disallow: /messages Disallow: /inventory Disallow: /c/*layout=list* # Slow and expensive. Disallow: /c/*sort=random* Disallow: /search* # Don't want to store sold pages. And is combinatorial. Disallow: *epoch=1* Disallow: *epoch=2* # Produces huge number of combos we don't care about. Disallow: /c/*store=* User-agent: Yandex Disallow: / # Produce lots of FE exceptions and also sell our data to competitors User-agent: AhrefsBot Disallow: / # Sells data to our competitors, thereby revealing secrets User-agent: SemrushBot Disallow: / Sitemap: https://www.morphmarket.com/sitemap.xml # Disallow: /stores # Disallow: /snakes # # User-agent: Googlebot # Allow: / # # User-agent: Bingbot # Allow: / # # User-agent: Teoma # Allow: / # # User-agent: Slurp # Allow: / # # User-agent: Facebot # Allow: /