Add robots.txt

Disallow all crawling.
Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>

patchwork/templates/robots.txt

+User-agent: *
+Disallow: /

patchwork/urls.py

@@ -24,6 +24,7 @@ from django.contrib import admin
 from django.contrib.auth import views as auth_views
 from rest_framework_nested import routers
 from patchwork.views.series import SeriesListView, SeriesView
+from django.views.generic import TemplateView
 import patchwork.views.api as api
 import patchwork.views
 import patchwork.views.bundle
@@ -77,6 +78,8 @@ patch_results_router.register(r'test-results', api.PatchResultViewSet,
 admin.autodiscover()
 
 urlpatterns = [
+    path('robots.txt', TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), name="robots_file"),
+
     path('admin/', admin.site.urls),
 
     # API