Add documentation, advancedAPI & fix info on admin page (1ad00501) · Commits · Zhang Yushan / cve-search

web/advanced_api.py

+97 −20

Original line number	Diff line number	Diff line
		@@ -12,15 +12,19 @@
		# imports
		import json
		import os
		import subprocess
		import sys
		_runPath = os.path.dirname(os.path.realpath(__file__))
		sys.path.append(os.path.join(_runPath, ".."))

		import lib.DatabaseLayer as db
		import sbin.db_blacklist as bl
		import sbin.db_whitelist as wl

		from bson import json_util
		from flask import Response, request
		from flask import Response, request, render_template
		from functools import wraps
		from io import StringIO

		from lib.Authentication import AuthenticationHandler
		from lib.User import User
		@@ -31,43 +35,116 @@ class Advanced_API(API):
		def __init__(self):
		super().__init__()
		self.auth_handler = AuthenticationHandler()
		routes = [{'r': '/api/admin/whitelist', 'm': ['GET'], 'f': self.api_admin_whitelist},]
		routes = [{'r': '/api/admin/whitelist', 'm': ['GET'], 'f': self.api_admin_whitelist},
		{'r': '/api/admin/blacklist', 'm': ['GET'], 'f': self.api_admin_blacklist},
		{'r': '/api/admin/whitelist/export', 'm': ['GET'], 'f': self.api_admin_whitelist},
		{'r': '/api/admin/blacklist/export', 'm': ['GET'], 'f': self.api_admin_blacklist},
		{'r': '/api/admin/whitelist/import', 'm': ['PUT'], 'f': self.api_admin_import_whitelist},
		{'r': '/api/admin/blacklist/import', 'm': ['PUT'], 'f': self.api_admin_import_blacklist},
		{'r': '/api/admin/whitelist/drop', 'm': ['POST'], 'f': self.api_admin_drop_whitelist},
		{'r': '/api/admin/blacklist/drop', 'm': ['POST'], 'f': self.api_admin_drop_blacklist},
		{'r': '/api/admin/whitelist/add', 'm': ['PUT'], 'f': self.api_admin_add_whitelist},
		{'r': '/api/admin/blacklist/add', 'm': ['PUT'], 'f': self.api_admin_add_blacklist},
		{'r': '/api/admin/whitelist/remove', 'm': ['PUT'], 'f': self.api_admin_remove_whitelist},
		{'r': '/api/admin/blacklist/remove', 'm': ['PUT'], 'f': self.api_admin_remove_blacklist},
		{'r': '/api/admin/updatedb', 'm': ['GET'], 'f': self.api_update_db}]

		for route in routes: self.addRoute(route)


		#############
		# Decorator #
		#############
		def token_required(funct):
		@wraps(funct)
		def api_token(args, *kwargs):
		def authErrors():
		# Check auth
		if not request.headers.get('Authorization'):
		return ({'status': 'error', 'reason': 'Authentication needed'}, 401)
		method, auth = (request.headers.get('Authorization')+" ").split(" ", 1) # Adding and removing space to ensure decent split
		data = None
		if method.lower() not in ['basic', 'token']:
		data = {'status': 'error', 'reason': 'Authorization method not allowed'}
		data = ({'status': 'error', 'reason': 'Authorization method not allowed'}, 400)
		else:
		try:
		name, token = (':'+auth.strip()).rsplit(":", 1)
		name = name[1:] # Adding and removing colon to ensure decent split
		if method.lower() == 'basic':
		data = {'status': 'error', 'reason': 'Authorization method not yet implemented'}
		data = ({'status': 'error', 'reason': 'Authorization method not yet implemented'}, 501)
		elif method.lower() == 'token':
		if not db.getToken(name) == token: data = {'status': 'error', 'reason': 'Authentication failed'}
		if not db.getToken(name) == token: data = ({'status': 'error', 'reason': 'Authentication failed'}, 401)
		except Exception as e:
		print(e)
		data = {'status': 'error', 'reason': 'Malformed Authentication String'}
		data = ({'status': 'error', 'reason': 'Malformed Authentication String'}, 400)
		if data:
		return data
		else: return None

		def token_required(funct):
		@wraps(funct)
		def api_token(args, *kwargs):
		data = Advanced_API.authErrors()
		if data:
		data = json.dumps(data, indent=2, sort_keys=True, default=json_util.default)
		return Response(data, mimetype='application/json')
		return Response(json.dumps(data[0], indent=2, sort_keys=True, default=json_util.default), mimetype='application/json'), data[1]
		else: return API.api(funct)(args, *kwargs)
		return api_token

		# Overriding api_dbInfo to allow for logged-in users to get more info
		def api_dbInfo(self):
		errors = Advanced_API.authErrors()
		admin = False if errors and errors[0].get('reason') == "Authentication needed" else True
		return API.api(db.getDBStats)(admin)

		# Overriding api_documentation to show the documentation for these functions
		def api_documentation(self):
		return render_template('api.html', advanced = True)

		@token_required
		def api_admin_whitelist(self):
		return db.getWhitelist()

		@token_required
		def api_admin_blacklist(self):
		return db.getBlacklist()

		@token_required
		def api_admin_import_whitelist(self):
		return wl.importWhitelist(StringIO(request.data.decode("utf-8")))

		@token_required
		def api_admin_import_blacklist(self):
		return bl.importBlacklist(StringIO(request.data.decode("utf-8")))

		@token_required
		def api_admin_drop_whitelist(self):
		return wl.dropWhitelist()

		@token_required
		def api_admin_drop_blacklist(self):
		return bl.dropBlacklist()

		@token_required
		def api_admin_add_whitelist(self):
		return wl.insertWhitelist(request.form['cpe'], request.form['type'])

		@token_required
		def api_admin_add_blacklist(self):
		return bl.insertBlacklist(request.form['cpe'], request.form['type'])

		@token_required
		def api_admin_remove_whitelist(self):
		return wl.removeWhitelist(request.form['cpe'])

		@token_required
		def api_admin_remove_blacklist(self):
		return bl.removeBlacklist(request.form['cpe'])



		@token_required
		def api_update_db(self):
		process = subprocess.Popen([sys.executable, os.path.join(_runPath, "../sbin/db_updater.py"), "-civ"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
		out, err = process.communicate()
		return "%s\n\nErrors:\n%s"%(str(out,'utf-8'),str(err,'utf-8')) if err else str(out,'utf-8')

		if __name__ == '__main__':
		server = Advanced_API()
		server.start()

web/api.py

+14 −8

Original line number	Diff line number	Diff line
		@@ -7,7 +7,7 @@
		#

		# Copyright (c) 2013-2016 Alexandre Dulaunoy - a@foo.be
		# Copyright (c) 2014-2016 Pieter-Jan Moreels - pieterjan.moreels@gmail.com
		# Copyright (c) 2014-2017 Pieter-Jan Moreels - pieterjan.moreels@gmail.com

		# imports
		import json
		@@ -22,7 +22,7 @@ _runPath = os.path.dirname(os.path.realpath(__file__))
		sys.path.append(os.path.join(_runPath, ".."))

		from bson import json_util
		from flask import Flask, request, Response
		from flask import Flask, request, Response, render_template
		from functools import wraps
		from logging.handlers import RotatingFileHandler
		from redis import exceptions as redisExceptions
		@@ -48,7 +48,8 @@ class API():
		app.config['SECRET_KEY'] = str(random.getrandbits(256))

		def __init__(self):
		routes = [{'r': '/api/cpe2.3/<path:cpe>', 'm': ['GET'], 'f': self.api_cpe23},
		routes = [{'r': '/api/', 'm': ['GET'], 'f': self.api_documentation},
		{'r': '/api/cpe2.3/<path:cpe>', 'm': ['GET'], 'f': self.api_cpe23},
		{'r': '/api/cpe2.2/<path:cpe>', 'm': ['GET'], 'f': self.api_cpe22},
		{'r': '/api/cvefor/<path:cpe>', 'm': ['GET'], 'f': self.api_cvesFor},
		{'r': '/api/cve/<cveid>', 'm': ['GET'], 'f': self.api_cve},
		@@ -59,7 +60,7 @@ class API():
		{'r': '/api/last/<int:limit>', 'm': ['GET'], 'f': self.api_last},
		{'r': '/api/browse', 'm': ['GET'], 'f': self.api_browse},
		{'r': '/api/browse/', 'm': ['GET'], 'f': self.api_browse},
		{'r': '/api/browse/<vendor>', 'm': ['GET'], 'f': self.api_browse},
		{'r': '/api/browse/<path:vendor>', 'm': ['GET'], 'f': self.api_browse},
		{'r': '/api/search/<vendor>/<path:product>', 'm': ['GET'], 'f': self.api_search},
		{'r': '/api/search/<path:search>', 'm': ['GET'], 'f': self.api_text_search},
		{'r': '/api/dbInfo', 'm': ['GET'], 'f': self.api_dbInfo}]
		@@ -80,7 +81,8 @@ class API():
		data = funct(args, *kwargs)
		except APIError as e:
		error = {'status': 'error', 'reason': e.message}
		except:
		except Exception as e:
		print(e)
		error = {'status': 'error', 'reason': 'Internal server error'}
		# Check if data should be returned as html or data
		try:
		@@ -97,6 +99,7 @@ class API():
		else:
		data = {'status': 'error', 'reason': 'Unknown Content-type requested'}
		returnType = 'application/json'
		if type(data) is not str:
		data = json.dumps(data, indent=2, sort_keys=True, default=json_util.default)
		return Response(data, mimetype=returnType)
		except:
		@@ -108,6 +111,10 @@ class API():
		##########
		# ROUTES #
		##########
		# /api
		def api_documentation(self):
		return render_template('api.html')

		# /api/cpe2.3/<cpe>
		@api
		def api_cpe23(self, cpe):
		@@ -124,8 +131,7 @@ class API():
		@api
		def api_cvesFor(self, cpe):
		cpe = urllib.parse.unquote_plus(cpe)
		cves = query.cvesForCPE(cpe)
		return cves
		return query.cvesForCPE(cpe)

		# /api/cve/<cveid>
		@api

web/index.py

+6 −6

Original line number	Diff line number	Diff line
		@@ -37,17 +37,17 @@ from lib.Config import Configuration
		from lib.PluginManager import PluginManager
		from lib.User import User
		from web.minimal import Minimal
		from web.advanced_api import Advanced_API

		class Index(Minimal):
		class Index(Minimal, Advanced_API):
		#############
		# Variables #
		#############

		def __init__(self):
		# TODO: replace super with:
		# Advanced_API.__init__()
		# Minimal.__init__()
		super().__init__()
		# TODO: make auth handler and plugin manager singletons
		Advanced_API.__init__(self)
		Minimal.__init__(self)
		self.minimal = False
		self.auth_handler = AuthenticationHandler()
		self.plugManager = PluginManager()
		@@ -189,7 +189,7 @@ class Index(Minimal):


		def adminInfo(self, output=None):
		return {'stats': db.getDBStats(),
		return {'stats': db.getDBStats(True),
		'plugins': self.plugManager.getPlugins(),
		'updateOutput': self.filterUpdateField(output)}

web/templates/api.html

0 → 100644

+154 −0

Original line number	Diff line number	Diff line
		{% set api_set = ['cpe22', 'cpe23', 'cvefor', 'cve', 'cwe', 'capec', 'last', 'browse', 'search', 'dbInfo'] %}
		{% set api_admin_set = ['whitelist', 'blacklist', 'whitelist_export', 'blacklist_export', 'whitelist_import',
		'blacklist_import', 'whitelist_drop', 'blacklist_drop', 'whitelist_add', 'blacklist_add',
		'whitelist_remove', 'blacklist_remove', 'dbupdate'] %}
		<html lang="en">
		<head>
		<!-- metadata -->
		<meta charset="utf-8">
		<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
		<meta name="description" content="Common Vulnerability Exposure most recent entries"/>
		<meta name="author" content="http://github.com/cve-search/cve-search - cve-search"/>

		<!-- css -->
		<link href="/static/css/bootstrap.min.css" rel="stylesheet" />
		<link href="/static/css/style.css" rel="stylesheet" />
		<link href="/static/css/sidebar.css" rel="stylesheet" />

		<!-- favicon -->
		<link rel="shortcut icon" href="/static/img/favicon.ico" />

		<!-- javascript -->
		<!-- HTML5 shim, for IE6-8 support of HTML5 elements -->
		<!--[if lt IE 9]>
		<script src="/static/js/html5shiv.js"></script>
		<![endif]-->
		<script type="text/javascript" src="/static/js/jquery-1.11.2.min.js"></script>
		<script type="text/javascript" src="/static/js/bootstrap.min.js"></script>
		<script type="text/javascript" src="/static/js/custom/scripts.js"></script>

		<title>API Documentation - CVE-Search</title>
		<style>
		body{
		background-color: DFDFDF;
		}
		#sidebar{
		background-color: gray;
		float: left;
		width: 205px;
		position: fixed;
		overflow: hidden;
		height: 100%;
		}
		#sidebar_content{
		width: 220px;
		overflow-y: scroll;
		height: 100%;
		}
		nav {
		padding: 20px 15px;
		}
		nav ul {
		padding-left: 5px;
		}
		nav ul li {
		margin: 0;
		padding: 0;
		list-style-type: none;
		}
		nav a {
		color: white;
		}
		nav a:hover{
		color: #DDD;
		}
		#content{
		margin-left: 210px;
		}
		</style>
		</head>
		<body>
		<div class="container-fluid">
		<div class="row">
		<div class="col-sm-12">
		<!-- Body Start -->
		<!-- Sidebar Nav -->
		<nav id="sidebar">
		<div id="sidebar_content">
		<a href="#auth"> <h4>Authentication </h4></a>
		<a href="#headers"><h4>Optional headers</h4></a>
		<a href="#queries"><h4>API Queries </h4></a>
		<ul id="API_Sidebar">
		{% for id in api_set %}
		<li><a data-toggle="collapse" data-parent="#api_links" href="#{{ id }}">/api/{{ id }}</a></li>
		{% endfor %}
		{% if advanced %}
		{% for id in api_admin_set %}
		<li><a data-toggle="collapse" data-parent="#api_links" href="#{{ id }}">/api/admin/{{ id\|replace('_', '/') }}</a></li>
		{% endfor %}
		{% endif %}
		</ul>
		</div>
		</nav>
		<!-- Documentation -->
		<div id="content">
		<div id="auth" class="panel panel-default">
		<div class="panel-body">
		<h4>Authentication</h4>
		<p>
		Some API calls require authentication. Authentication is done in one of two ways:
		<ul>
		<li>Username - Password (Not recommended)</li>
		<li>Username - Token </li>
		</ul>
		Authentication is done by adding the following header to the HTTP request: <br />
		<pre>Authorization: token user:679c2955085b46e48155b84f4c878844</pre> <br />
		<b>PLEASE NOTE: Neither the password nor the token are obfuscated, so it is <u>strongly</u> advised to use HTTPS</b>
		</p>
		</div>
		</div>
		<div id="headers" class="panel panel-default">
		<div class="panel-body">
		<h4>Optional Headers</h4>
		The following headers can be appended to any request:
		<ul>
		<li>Accept</li>
		<li>Version</li>
		</ul>
		<h5><b>Accept</b></h5>
		The Accept argument may contain one of two categories:
		<ul>
		<li>/json (/* will default to text/json)</li>
		<li>*/plain</li>
		</ul>
		All the examples you see in this documentation are the output of the */plain choice. <br/>
		The */json choice will incapsulate all output with a status code, of the format:
		<pre>{'status': 'success', 'data': <output of */plain>}</pre>
		<h5><b>Version</b></h5>
		The version of the API call. For backwards compatibility, when the version is not specified, version 1.0 (legacy) will be used
		and only plain text output will be used. As of version 1.1, the Accept header will be taken into account.
		</div>
		</div>
		<div id="queries" class="panel panel-default">
		<div class="panel-body">
		<h4>API Queries</h4>
		<div class="panel-group" id="api_links">
		{% for doc in api_set %}
		{% include 'documentation/api/'+doc%}
		{% endfor %}
		{% if advanced %}
		{% for doc in api_admin_set %}
		{% include 'documentation/api/'+doc%}
		{% endfor %}
		{% endif %}
		</div>
		</div>
		</div>
		</div>
		<a href="#" class="back-to-top">Back to Top</a>
		<!-- Body End -->
		</div>
		</div>
		</div>
		</body>
		</html>

web/templates/documentation/api/blacklist

0 → 100644

+36 −0

Original line number	Diff line number	Diff line
		{% extends 'layouts/api-accordion-child' %}
		{% set admin = True %}
		{% set id = "blacklist" %}
		{% set title = "/api/admin/blacklist" %}
		{% set method = "GET" %}


		{% block desc %}
		Returns the content of the blacklist. <br />
		The blacklist is a list of CPEs that will hide a CVE when all the CPEs a product applies to are blacklisted.
		It is intended to be used as a way to hide unwanted information. <br />
		There are three types of CPEs:
		<ul>
		<li>cpe - A fully qualified CPE code in CPE2.2 or CPE2.3 format</li>
		<li>targetsoftware - A software the CPE applies to. Equivalent to cpe:2.3:-:-:-:-:-:-:-:-:<cpe></li>
		<li>targethardware - A hardware the CPE applies to. Equivalent to cpe:2.3:-:-:-:-:-:-:-:-:-:<cpe></li>
		</ul>
		Other types are possible, but are not used by the software.
		{% endblock %}

		{% block output %}
		[
		{
		"id": "iphone_os",
		"type": "targetsoftware"
		},
		{
		"id": "cpe:2.3:o:apple",
		"type": "cpe"
		},
		{
		"id": "cpe:2.3:o:microsoft:windows_xp",
		"type": "cpe"
		}
		]
		{% endblock %}

Admin message