diff --git a/js/admin.js b/js/admin.js
index 0e561fa..cdb39ae 100644
--- a/js/admin.js
+++ b/js/admin.js
@@ -70,6 +70,22 @@ $(function() {
 		});
 	});
 
+	$('#user-saml').change(function() {
+		// Checks on each request whether the settings make sense or not
+		$.ajax({
+			url: OC.generateUrl('/apps/user_saml/saml/metadata'),
+			type: 'GET'
+		}).fail(function (e) {
+			if(e.status === 500) {
+				$('#user-saml-settings-complete').addClass('hidden');
+				$('#user-saml-settings-incomplete').removeClass('hidden');
+			}
+		}).success(function (e) {
+			$('#user-saml-settings-complete').removeClass('hidden');
+			$('#user-saml-settings-incomplete').addClass('hidden');
+		})
+	});
+
 	$('#user-saml-settings .toggle').on('click', function() {
 		var el = $(this),
 			nextSibling = el.parent().next(),
diff --git a/templates/admin.php b/templates/admin.php
index d5282ba..a0ce55e 100644
--- a/templates/admin.php
+++ b/templates/admin.php
@@ -80,7 +80,7 @@ style('user_saml', 'admin');
 		</div>
 
 		<a href="<?php p(\OC::$server->getURLGenerator()->linkToRoute('user_saml.SAML.getMetadata')) ?>" class="button"><?php p($l->t('Download metadata XML')) ?></a>
-		<!-- FIXME: Add test settings -->
-		<a class="button"><?php p($l->t('Test settings')) ?></a>
+		<span class="warning hidden" id="user-saml-settings-incomplete"><?php p($l->t('Metadata invalid')) ?></span>
+		<span class="success hidden" id="user-saml-settings-complete"><?php p($l->t('Metadata valid')) ?></span>
 	</div>
 </form>