archive
/
nextcloud
mirror of https://github.com/nextcloud/server.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Change X-Robots-Tag header from "none" to "noindex, nofollow"

Browse Source 
While "none" is indeed equivalent to "noindex, nofollow" for Google, but seems to be not supported by Bing and probably other search engines.

https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta/name#other_metadata_names
https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag?hl=de#comma-separated-list
https://www.bing.com/webmasters/help/which-robots-metatags-does-bing-support-5198d240

Signed-off-by: MichaIng <micha@dietpi.com>
pull/36689/head
MichaIng 1 year ago
parent b36a31c918
commit 5f90b8eb11
No known key found for this signature in database
GPG Key ID: 0442B9ADE65643FE
12 changed files with 80 additions and 37 deletions
Show Stats Download Patch File Download Diff File

2
.htaccess
Unescape Escape View File

@ -31,7 +31,7 @@
Header always set X-Permitted-Cross-Domain-Policies "none"
Header onsuccess unset X-Robots-Tag
Header always set X-Robots-Tag "none"
Header always set X-Robots-Tag "noindex, nofollow"
Header onsuccess unset X-XSS-Protection
Header always set X-XSS-Protection "1; mode=block"

4
build/integration/features/carddav.feature
Unescape Escape View File

@ -46,7 +46,7 @@ Feature: carddav
|X-Content-Type-Options |nosniff|
|X-Frame-Options|SAMEORIGIN|
|X-Permitted-Cross-Domain-Policies|none|
|X-Robots-Tag|none|
|X-Robots-Tag|noindex, nofollow|
|X-XSS-Protection|1; mode=block|
Scenario: Exporting the picture of ones own contact
@ -60,5 +60,5 @@ Feature: carddav
|X-Content-Type-Options |nosniff|
|X-Frame-Options|SAMEORIGIN|
|X-Permitted-Cross-Domain-Policies|none|
|X-Robots-Tag|none|
|X-Robots-Tag|noindex, nofollow|
|X-XSS-Protection|1; mode=block|

2
build/integration/features/dav-v2.feature
Unescape Escape View File

@ -27,7 +27,7 @@ Feature: dav-v2
|X-Content-Type-Options |nosniff|
|X-Frame-Options|SAMEORIGIN|
|X-Permitted-Cross-Domain-Policies|none|
|X-Robots-Tag|none|
|X-Robots-Tag|noindex, nofollow|
|X-XSS-Protection|1; mode=block|
And Downloaded content should start with "Welcome to your Nextcloud account!"

4
build/integration/features/webdav-related.feature
Unescape Escape View File

@ -53,7 +53,7 @@ Feature: webdav-related
And User "user0" moves file "/textfile0.txt" to "/testshare/textfile0.txt"
And the HTTP status code should be "403"
When Downloading file "/testshare/textfile0.txt"
Then the HTTP status code should be "404"
Then the HTTP status code should be "404"
Scenario: Moving a file to overwrite a file in a folder with no permissions
Given using old dav path
@ -251,7 +251,7 @@ Feature: webdav-related
|X-Content-Type-Options |nosniff|
|X-Frame-Options|SAMEORIGIN|
|X-Permitted-Cross-Domain-Policies|none|
|X-Robots-Tag|none|
|X-Robots-Tag|noindex, nofollow|
|X-XSS-Protection|1; mode=block|
And Downloaded content should start with "Welcome to your Nextcloud account!"

4
core/js/setupchecks.js
Unescape Escape View File

@ -628,13 +628,13 @@
if (xhr.status === 200) {
var securityHeaders = {
'X-Content-Type-Options': ['nosniff'],
'X-Robots-Tag': ['none'],
'X-Robots-Tag': ['noindex, nofollow'],
'X-Frame-Options': ['SAMEORIGIN', 'DENY'],
'X-Permitted-Cross-Domain-Policies': ['none'],
};
for (var header in securityHeaders) {
var option = securityHeaders[header][0];
if(!xhr.getResponseHeader(header) || xhr.getResponseHeader(header).toLowerCase() !== option.toLowerCase()) {
if(!xhr.getResponseHeader(header) || xhr.getResponseHeader(header).replace(/, /, ',').toLowerCase() !== option.replace(/, /, ',').toLowerCase()) {
var msg = t('core', 'The "{header}" HTTP header is not set to "{expected}". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.', {header: header, expected: option});
if(xhr.getResponseHeader(header) && securityHeaders[header].length > 1 && xhr.getResponseHeader(header).toLowerCase() === securityHeaders[header][1].toLowerCase()) {
msg = t('core', 'The "{header}" HTTP header is not set to "{expected}". Some features might not work correctly, as it is recommended to adjust this setting accordingly.', {header: header, expected: option});

89
core/js/tests/specs/setupchecksSpec.js
Unescape Escape View File

@ -1569,7 +1569,7 @@ describe('OC.SetupChecks tests', function() {
msg: 'The "X-Content-Type-Options" HTTP header is not set to "nosniff". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
type: OC.SetupChecks.MESSAGE_TYPE_WARNING
}, {
msg: 'The "X-Robots-Tag" HTTP header is not set to "none". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
msg: 'The "X-Robots-Tag" HTTP header is not set to "noindex, nofollow". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
type: OC.SetupChecks.MESSAGE_TYPE_WARNING
}, {
msg: 'The "X-Frame-Options" HTTP header is not set to "SAMEORIGIN". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
@ -1596,7 +1596,7 @@ describe('OC.SetupChecks tests', function() {
suite.server.requests[0].respond(
200,
{
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'Strict-Transport-Security': 'max-age=15768000;preload',
'X-Permitted-Cross-Domain-Policies': 'none',
@ -1627,7 +1627,7 @@ describe('OC.SetupChecks tests', function() {
{
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'Strict-Transport-Security': 'max-age=15768000',
'X-Permitted-Cross-Domain-Policies': 'none',
@ -1641,6 +1641,49 @@ describe('OC.SetupChecks tests', function() {
});
});
describe('check X-Robots-Tag header', function() {
it('should return no message if X-Robots-Tag is set to noindex,nofollow without space', function(done) {
protocolStub.returns('https');
var result = OC.SetupChecks.checkGeneric();
suite.server.requests[0].respond(200, {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'noindex,nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
});
result.done(function( data, s, x ){
expect(data).toEqual([]);
done();
});
});
it('should return a message if X-Robots-Tag is set to none', function(done) {
protocolStub.returns('https');
var result = OC.SetupChecks.checkGeneric();
suite.server.requests[0].respond(200, {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
});
result.done(function( data, s, x ){
expect(data).toEqual([
{
msg: 'The "X-Robots-Tag" HTTP header is not set to "noindex, nofollow". This is a potential security or privacy risk, as it is recommended to adjust this setting accordingly.',
type: OC.SetupChecks.MESSAGE_TYPE_WARNING
}
]);
done();
});
});
});
describe('check X-XSS-Protection header', function() {
it('should return no message if X-XSS-Protection is set to 1; mode=block; report=https://example.com', function(done) {
protocolStub.returns('https');
@ -1650,7 +1693,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block; report=https://example.com',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1670,7 +1713,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1690,7 +1733,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1715,7 +1758,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '0',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1742,7 +1785,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1762,7 +1805,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer-when-downgrade',
@ -1782,7 +1825,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'strict-origin',
@ -1802,7 +1845,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'strict-origin-when-cross-origin',
@ -1822,7 +1865,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'same-origin',
@ -1842,7 +1885,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'origin',
@ -1867,7 +1910,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'origin-when-cross-origin',
@ -1892,7 +1935,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'unsafe-url',
@ -1919,7 +1962,7 @@ describe('OC.SetupChecks tests', function() {
{
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1965,7 +2008,7 @@ describe('OC.SetupChecks tests', function() {
{
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -1990,7 +2033,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15551999',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2015,7 +2058,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'iAmABogusHeader342',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2039,7 +2082,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=15768000',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2059,7 +2102,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=99999999',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2079,7 +2122,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=99999999; includeSubDomains',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',
@ -2099,7 +2142,7 @@ describe('OC.SetupChecks tests', function() {
'Strict-Transport-Security': 'max-age=99999999; preload; includeSubDomains',
'X-XSS-Protection': '1; mode=block',
'X-Content-Type-Options': 'nosniff',
'X-Robots-Tag': 'none',
'X-Robots-Tag': 'noindex, nofollow',
'X-Frame-Options': 'SAMEORIGIN',
'X-Permitted-Cross-Domain-Policies': 'none',
'Referrer-Policy': 'no-referrer',

2
lib/private/legacy/OC_Response.php
Unescape Escape View File

@ -99,7 +99,7 @@ class OC_Response {
header('X-Content-Type-Options: nosniff'); // Disable sniffing the content type for IE
header('X-Frame-Options: SAMEORIGIN'); // Disallow iFraming from other domains
header('X-Permitted-Cross-Domain-Policies: none'); // https://www.adobe.com/devnet/adobe-media-server/articles/cross-domain-xml-for-streaming.html
header('X-Robots-Tag: none'); // https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag
header('X-Robots-Tag: noindex, nofollow'); // https://developers.google.com/webmasters/control-crawl-index/docs/robots_meta_tag
header('X-XSS-Protection: 1; mode=block'); // Enforce browser based XSS filters
}
}

2
lib/public/AppFramework/Http/Response.php
Unescape Escape View File

@ -257,7 +257,7 @@ class Response {
$this->headers['Content-Security-Policy'] = $this->getContentSecurityPolicy()->buildPolicy();
$this->headers['Feature-Policy'] = $this->getFeaturePolicy()->buildPolicy();
$this->headers['X-Robots-Tag'] = 'none';
$this->headers['X-Robots-Tag'] = 'noindex, nofollow';
if ($this->ETag) {
$mergeWith['ETag'] = '"' . $this->ETag . '"';

2
tests/data/setUploadLimit/htaccess
Unescape Escape View File

@ -11,7 +11,7 @@
# Add security and privacy related headers
Header set X-Content-Type-Options "nosniff"
Header set X-XSS-Protection "1; mode=block"
Header set X-Robots-Tag "none"
Header set X-Robots-Tag "noindex, nofollow"
Header set X-Frame-Options "SAMEORIGIN"
SetEnv modHeadersAvailable true
</IfModule>

2
tests/lib/AppFramework/Controller/ControllerTest.php
Unescape Escape View File

@ -116,7 +116,7 @@ class ControllerTest extends \Test\TestCase {
'Content-Security-Policy' => "default-src 'none';base-uri 'none';manifest-src 'self';frame-ancestors 'none'",
'Feature-Policy' => "autoplay 'none';camera 'none';fullscreen 'none';geolocation 'none';microphone 'none';payment 'none'",
'X-Request-Id' => $this->request->getId(),
'X-Robots-Tag' => 'none',
'X-Robots-Tag' => 'noindex, nofollow',
];
$response = $this->controller->customDataResponse(['hi']);

2
tests/lib/AppFramework/Http/DataResponseTest.php
Unescape Escape View File

@ -67,7 +67,7 @@ class DataResponseTest extends \Test\TestCase {
'Cache-Control' => 'no-cache, no-store, must-revalidate',
'Content-Security-Policy' => "default-src 'none';base-uri 'none';manifest-src 'self';frame-ancestors 'none'",
'Feature-Policy' => "autoplay 'none';camera 'none';fullscreen 'none';geolocation 'none';microphone 'none';payment 'none'",
'X-Robots-Tag' => 'none',
'X-Robots-Tag' => 'noindex, nofollow',
'X-Request-Id' => \OC::$server->get(IRequest::class)->getId(),
];
$expectedHeaders = array_merge($expectedHeaders, $headers);

2
tests/lib/AppFramework/Http/ResponseTest.php
Unescape Escape View File

@ -51,7 +51,7 @@ class ResponseTest extends \Test\TestCase {
'Last-Modified' => 1,
'ETag' => 3,
'Something-Else' => 'hi',
'X-Robots-Tag' => 'none',
'X-Robots-Tag' => 'noindex, nofollow',
];
$this->childResponse->setHeaders($expected);

Write Preview
Loading…
Cancel
Save