From 57f8dea4e02c33bb970f7a094ca9b361d7a7601c Mon Sep 17 00:00:00 2001 From: Marian Steinbach Date: Wed, 3 Oct 2018 21:01:52 +0200 Subject: [PATCH] Improve certificate check to support SNI (#71) * Fix the certificate check to support SNI * Better tests for the certificate check * Activate verbose output when running make test * Add commenting on the spider test --- Makefile | 2 +- checks/certificate.py | 44 +++++++++++++++++++++++++++----------- checks/certificate_test.py | 35 ++++++++++++++++++++++++++++++ spider/spider_test.py | 13 ++++++++--- 4 files changed, 78 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 9b85fa4..0481f06 100644 --- a/Makefile +++ b/Makefile @@ -44,5 +44,5 @@ test: dockerimage docker run --rm -ti \ --entrypoint "python3" \ $(IMAGE) \ - -m unittest discover -p '*_test.py' + -m unittest discover -p '*_test.py' -v diff --git a/checks/certificate.py b/checks/certificate.py index 2539963..e00ea7f 100644 --- a/checks/certificate.py +++ b/checks/certificate.py @@ -4,6 +4,7 @@ Gathers information on the TLS/SSL certificate used by a server from urllib.parse import urlparse import logging +import socket import ssl from datetime import datetime from datetime import timezone @@ -36,21 +37,40 @@ class Checker(AbstractChecker): } parsed = urlparse(url) + port = 443 + if parsed.port is not None: + port = parsed.port + try: - cert = ssl.get_server_certificate((parsed.hostname, 443)) - x509 = crypto.load_certificate(crypto.FILETYPE_PEM, cert) - result['serial_number'] = str(x509.get_serial_number()) + #cert = ssl.get_server_certificate((parsed.hostname, port)) - nb = x509.get_notBefore().decode('utf-8') - na = x509.get_notAfter().decode('utf-8') - - # parse '2018 06 27 00 00 00Z' - result['not_before'] = datetime(int(nb[0:4]), int(nb[4:6]), int(nb[6:8]), int(nb[8:10]), int(nb[10:12]), int(nb[12:14]), tzinfo=timezone.utc).isoformat() - result['not_after'] = datetime(int(na[0:4]), int(na[4:6]), int(na[6:8]), int(na[8:10]), int(na[10:12]), int(na[12:14]), tzinfo=timezone.utc).isoformat() + context = ssl.create_default_context() - # decode and convert from bytes to unicode - result['subject'] = dict([tuple(map(lambda x: x.decode('utf-8'), tup)) for tup in x509.get_subject().get_components()]) - result['issuer'] = dict([tuple(map(lambda x: x.decode('utf-8'), tup)) for tup in x509.get_issuer().get_components()]) + # get certificate with SNI + with socket.create_connection((parsed.hostname, port)) as sock: + with context.wrap_socket(sock, server_hostname=parsed.hostname) as sslsock: + der_cert = sslsock.getpeercert(True) + cert = ssl.DER_cert_to_PEM_cert(der_cert) + + try: + x509 = crypto.load_certificate(crypto.FILETYPE_PEM, cert) + result['serial_number'] = str(x509.get_serial_number()) + + nb = x509.get_notBefore().decode('utf-8') + na = x509.get_notAfter().decode('utf-8') + + # parse '2018 06 27 00 00 00Z' + result['not_before'] = datetime(int(nb[0:4]), int(nb[4:6]), int(nb[6:8]), int(nb[8:10]), int(nb[10:12]), int(nb[12:14]), tzinfo=timezone.utc).isoformat() + result['not_after'] = datetime(int(na[0:4]), int(na[4:6]), int(na[6:8]), int(na[8:10]), int(na[10:12]), int(na[12:14]), tzinfo=timezone.utc).isoformat() + + # decode and convert from bytes to unicode + result['subject'] = dict([tuple(map(lambda x: x.decode('utf-8'), tup)) for tup in x509.get_subject().get_components()]) + result['issuer'] = dict([tuple(map(lambda x: x.decode('utf-8'), tup)) for tup in x509.get_issuer().get_components()]) + except Exception as e: + result['exception'] = { + 'type': str(type(e)), + 'message': str(e), + } except Exception as e: result['exception'] = { diff --git a/checks/certificate_test.py b/checks/certificate_test.py index 66c2288..53588fd 100644 --- a/checks/certificate_test.py +++ b/checks/certificate_test.py @@ -1,10 +1,13 @@ from checks import certificate from checks.config import Config + import unittest +from pprint import pprint class TestCertificateChecker(unittest.TestCase): def test_google(self): + """Load cert from a site that should work""" url = 'https://www.google.com/' config = Config(urls=[url]) checker = certificate.Checker(config=config, previous_results={}) @@ -14,6 +17,7 @@ class TestCertificateChecker(unittest.TestCase): self.assertEqual(result[url]['issuer']['O'], 'Google Trust Services') def test_kaarst(self): + """Real-workd example""" url = 'https://www.gruenekaarst.de/' config = Config(urls=[url]) checker = certificate.Checker(config=config, previous_results={}) @@ -22,6 +26,37 @@ class TestCertificateChecker(unittest.TestCase): self.assertIsNone(result[url]['exception']) self.assertEqual(result[url]['issuer']['O'], 'COMODO CA Limited') + def test_tls_v_1_0(self): + """Load a certificate for a TLS v1.0 server""" + url = 'https://tls-v1-0.badssl.com:1010/' + config = Config(urls=[url]) + checker = certificate.Checker(config=config, previous_results={}) + result = checker.run() + self.assertIn(url, result) + self.assertIsNone(result[url]['exception']) + self.assertEqual(result[url]['subject']['CN'], '*.badssl.com') + + def test_tls_v_1_1(self): + """Load a certificate for a TLS v1.1 server""" + url = 'https://tls-v1-1.badssl.com:1011/' + config = Config(urls=[url]) + checker = certificate.Checker(config=config, previous_results={}) + result = checker.run() + self.assertIn(url, result) + self.assertIsNone(result[url]['exception']) + self.assertEqual(result[url]['subject']['CN'], '*.badssl.com') + + def test_tls_v_1_2(self): + """Load a certificate for a TLS v1.2 server""" + url = 'https://tls-v1-2.badssl.com:1012/' + config = Config(urls=[url]) + checker = certificate.Checker(config=config, previous_results={}) + result = checker.run() + self.assertIn(url, result) + self.assertIsNone(result[url]['exception']) + self.assertEqual(result[url]['subject']['CN'], '*.badssl.com') + + if __name__ == '__main__': unittest.main() diff --git a/spider/spider_test.py b/spider/spider_test.py index dda55e7..cc10402 100644 --- a/spider/spider_test.py +++ b/spider/spider_test.py @@ -1,12 +1,19 @@ import unittest +from pprint import pprint from spider.spider import check_and_rate_site -from pprint import pprint +class TestSpider(unittest.TestCase): -class TestSpiderr(unittest.TestCase): + """ + Simply calls the spider.check_and_rate_site function + with httpbin.org URLs. We don't assert a lot here, + but at least make sure that most of our code is executed + in tests. + """ - def test_url1(self): + def test_html(self): + """Loads a simple HTML web page""" entry = { "url": "https://httpbin.org/html",