Merge pull request #69 from adulau/master (cd0b308a) · Commits · Zhang Yushan / cve-search

bin/search_fulltext.py

+2 −3

Original line number	Diff line number	Diff line
		@@ -5,7 +5,7 @@
		#
		# Software is free software released under the "Modified BSD license"
		#
		# Copyright (c) 2012-2014 Alexandre Dulaunoy - a@foo.be
		# Copyright (c) 2012-2015 Alexandre Dulaunoy - a@foo.be
		# Copyright (c) 2015 Pieter-Jan Moreels - pieterjan.moreels@gmail.com

		# Imports
		@@ -20,7 +20,6 @@ import argparse
		import json
		from bson import json_util


		runPath = os.path.dirname(os.path.realpath(__file__))

		indexpath = "./indexdir"
		@@ -32,7 +31,7 @@ argParser = argparse.ArgumentParser(description='Full text search for cve-search
		argParser.add_argument('-q', action='append', help='query to lookup (one or more)')
		argParser.add_argument('-t', action='store_true', help='output title of the match CVE(s)')
		argParser.add_argument('-f', action='store_true', help='output matching CVE(s) in JSON')
		argParser.add_argument('-m', type=int, default=False, help='most frequent terms)')
		argParser.add_argument('-m', type=int, default=False, help='most frequent terms in CVE description (m is top-m values)')
		argParser.add_argument('-l', action='store_true', default=False, help='dump all terms encountered in CVE description')
		argParser.add_argument('-g', action='store_true', default=False, help='graph of most frequent terms with each matching CVE (JSON output)')
		argParser.add_argument('-s', action='store_true', default=False, help='enable stemming on graph JSON output (default is False)')

lib/Query.py

+14 −3

Original line number	Diff line number	Diff line
		@@ -5,12 +5,17 @@
		#
		# Software is free software released under the "Modified BSD license"
		#
		# Copyright (c) 2014-2015 Alexandre Dulaunoy - a@foo.be
		# Copyright (c) 2014-2015 Pieter-Jan Moreels - pieterjan.moreels@gmail.com

		import urllib.parse
		import json
		import requests

		import os
		import sys
		runPath = os.path.dirname(os.path.realpath(__file__))
		sys.path.append(os.path.join(runPath, ".."))

		from lib.Config import Configuration


		@@ -50,9 +55,10 @@ def lookupcpe(cpeid=None):
		return e['title']


		def lastentries(limit=5, namelookup=False):
		def lastentries(limit=5, namelookup=False, rankinglookup=True):
		entries = []
		for item in collection.find({}).sort("Modified", -1).limit(limit):
		item.pop('_id')
		if not namelookup and rankinglookup is not True:
		entries.append(item)
		else:
		@@ -69,7 +75,7 @@ def lastentries(limit=5, namelookup=False):
		if rank and rank not in ranking:
		ranking.append(rank)
		item['vulnerable_configuration'] = vulconf
		if rankinglookup:
		if rankinglookup and len(ranking) > 0:
		item['ranking'] = ranking
		entries.append(item)
		return entries
		@@ -109,3 +115,8 @@ def apisearch(api, query=None):
		return r.text
		else:
		return False

		def _test():
		print (lastentries(limit=10))
		if __name__ == "__main__":
		_test()

sbin/db_fulltext.py

+7 −5

Original line number	Diff line number	Diff line
		@@ -32,7 +32,7 @@ collection = db.cves

		argParser = argparse.ArgumentParser(description='Fulltext indexer for the MongoDB CVE collection')
		argParser.add_argument('-v', action='store_true', default=False, help='Verbose logging')
		argParser.add_argument('-l', default=None, help='Number of last entries to index')
		argParser.add_argument('-l', default=5, help='Number of last entries to index (Default: 5) - 0 to index all documents')
		argParser.add_argument('-n', action='store_true', default=False, help='lookup complete cpe (Common Platform Enumeration) name for vulnerable configuration to add in the index')
		args = argParser.parse_args()

		@@ -52,8 +52,6 @@ if not exists_in(indexpath):
		ix = create_in(indexpath, schema)
		else:
		ix = open_dir(indexpath)


		def dumpallcveid(entry=None):
		cveid = []
		if entry is None:
		@@ -71,7 +69,11 @@ def getcve(cveid=None):
		return collection.find_one({'id': cveid})

		for cveid in progressbar(dumpallcveid(entry=args.l),prefix="Processing"):
		try:
		writer = ix.writer()
		except:
		print ("Index is locked. Another db_fulltext process running?")
		sys.exit(1)
		item = getcve(cveid=cveid)
		title = item['summary'][0:70]
		if args.n:

sbin/db_mgmt_admin.py

+4 −1

Original line number	Diff line number	Diff line
		@@ -45,7 +45,8 @@ exits = {'userInDb': 'User already exists in database',
		'userpasscombo': 'Master user/password combination does not exist',
		'passwordMatch': "The passwords don't match!",
		'noMaster': 'Not a master account!',
		'lastMaster': 'This user is the last admin in the database and thus can not be removed'}
		'lastMaster': 'This user is the last admin in the database and thus can not be removed',
		'dummy': '_dummy_ is a placeholder, and thus cannot be used'}

		# functions

		@@ -89,6 +90,8 @@ def isLastAdmin(user):
		try:
		if args.a:
		username = args.a
		if username.strip() == "_dummy_":
		sys.exit(exits['dummy'])
		if existsInDB(username):
		sys.exit(exits['userInDb'])
		# set master if db is empty

web/index.py

+84 −37

Original line number	Diff line number	Diff line
		@@ -134,6 +134,11 @@ def blacklist_mark(cve):
		return cve


		def seen_mark(cve):
		seen=getSeenCVEs()
		for c in cve:
		if c["id"] in seen: cve[cve.index(c)]['seen'] = 'yes'

		def getBlacklist():
		collection = db.mgmt_blacklist
		blacklist = collection.find()
		@@ -158,6 +163,17 @@ def getBlacklistRegexes():
		return regexes


		def getSeenCVEs():
		cu=current_user.get_id()
		collection = db.mgmt_seen
		userdata = collection.find({"user":cu})
		if userdata.count()==0:
		collection.insert({"user":cu, "seen_cves":[]})
		return []
		else:
		return userdata[0]["seen_cves"]


		def addCPEToList(cpe, listType, cpeType=None):
		if not cpeType:
		cpeType='cpe'
		@@ -200,7 +216,7 @@ def adminStats():


		def filter_logic(blacklist, whitelist, unlisted, timeSelect, startDate, endDate,
		timeTypeSelect, cvssSelect, cvss, rejectedSelect, limit, skip):
		timeTypeSelect, cvssSelect, cvss, rejectedSelect, hideSeen, limit, skip):
		collection = db.cves
		query = []
		# retrieving lists
		@@ -228,6 +244,11 @@ def filter_logic(blacklist, whitelist, unlisted, timeSelect, startDate, endDate,
		if rejectedSelect == "hide":
		exp = "^(?!\\ REJECT \\\s+DO NOT USE THIS CANDIDATE NUMBER.*)"
		query.append({'summary': re.compile(exp)})

		if current_user.is_authenticated():
		if hideSeen == "hide":
		query.append({'id': {"$nin":getSeenCVEs()}})

		# cvss logic
		if cvssSelect != "all":
		if cvssSelect == "above":
		@@ -259,6 +280,7 @@ def filter_logic(blacklist, whitelist, unlisted, timeSelect, startDate, endDate,
		cve = whitelist_mark(cve)
		if blacklist == "mark":
		cve = blacklist_mark(cve)
		seen_mark(cve)
		cve = list(cve)
		return cve

		@@ -281,6 +303,28 @@ def markCPEs(cve):
		return cve


		def getFilterSettingsFromPost():
		blacklist = request.form.get('blacklistSelect')
		whitelist = request.form.get('whitelistSelect')
		unlisted = request.form.get('unlistedSelect')
		timeSelect = request.form.get('timeSelect')
		startDate = request.form.get('startDate')
		endDate = request.form.get('endDate')
		timeTypeSelect = request.form.get('timeTypeSelect')
		cvssSelect = request.form.get('cvssSelect')
		cvss = request.form.get('cvss')
		rejectedSelect = request.form.get('rejectedSelect')
		hideSeen = request.form.get('hideSeen')
		settings = {'blacklistSelect': blacklist, 'whitelistSelect': whitelist,
		'unlistedSelect': unlisted, 'timeSelect': timeSelect,
		'startDate': startDate, 'endDate': endDate,
		'timeTypeSelect': timeTypeSelect, 'cvssSelect': cvssSelect,
		'cvss': cvss, 'rejectedSelect': rejectedSelect, "hideSeen": hideSeen}
		# retrieving data
		cve = filter_logic(blacklist, whitelist, unlisted, timeSelect, startDate, endDate,
		timeTypeSelect, cvssSelect, cvss, rejectedSelect, hideSeen, pageLength, 0)
		return(settings,cve)

		@login_manager.user_loader
		def load_user(id):
		return User.get(id)
		@@ -299,30 +343,14 @@ def index():
		cvssSelect = "all"
		cvss = None
		rejectedSelect = "hide"
		hideSeen = "show"
		cve = filter_logic(blacklist, whitelist, unlisted, timeSelect, startDate, endDate,
		timeTypeSelect, cvssSelect, cvss, rejectedSelect, pageLength, 0)
		timeTypeSelect, cvssSelect, cvss, rejectedSelect, hideSeen, pageLength, 0)
		return render_template('index.html', cve=cve, r=0, pageLength=pageLength)

		@app.route('/', methods=['POST'])
		def filterPost():
		blacklist = request.form.get('blacklistSelect')
		whitelist = request.form.get('whitelistSelect')
		unlisted = request.form.get('unlistedSelect')
		timeSelect = request.form.get('timeSelect')
		startDate = request.form.get('startDate')
		endDate = request.form.get('endDate')
		timeTypeSelect = request.form.get('timeTypeSelect')
		cvssSelect = request.form.get('cvssSelect')
		cvss = request.form.get('cvss')
		rejectedSelect = request.form.get('rejectedSelect')
		settings = {'blacklistSelect': blacklist, 'whitelistSelect': whitelist,
		'unlistedSelect': unlisted, 'timeSelect': timeSelect,
		'startDate': startDate, 'endDate': endDate,
		'timeTypeSelect': timeTypeSelect, 'cvssSelect': cvssSelect,
		'cvss': cvss, 'rejectedSelect': rejectedSelect}
		# retrieving data
		cve = filter_logic(blacklist, whitelist, unlisted, timeSelect, startDate, endDate,
		timeTypeSelect, cvssSelect, cvss, rejectedSelect, pageLength, 0)
		settings,cve = getFilterSettingsFromPost()
		return render_template('index.html', settings=settings, cve=cve, r=0, pageLength=pageLength)


		@@ -330,27 +358,35 @@ def filterPost():
		def filterLast(r):
		if not r:
		r = 0
		blacklist = request.form.get('blacklistSelect')
		whitelist = request.form.get('whitelistSelect')
		unlisted = request.form.get('unlistedSelect')
		timeSelect = request.form.get('timeSelect')
		startDate = request.form.get('startDate')
		endDate = request.form.get('endDate')
		timeTypeSelect = request.form.get('timeTypeSelect')
		cvssSelect = request.form.get('cvssSelect')
		cvss = request.form.get('cvss')
		rejectedSelect = request.form.get('rejectedSelect')
		settings = {'blacklistSelect': blacklist, 'whitelistSelect': whitelist,
		'unlistedSelect': unlisted, 'timeSelect': timeSelect,
		'startDate': startDate, 'endDate': endDate,
		'timeTypeSelect': timeTypeSelect, 'cvssSelect': cvssSelect,
		'cvss': cvss, 'rejectedSelect': rejectedSelect}
		settings,cve = getFilterSettingsFromPost()
		return render_template('index.html', settings=settings, cve=cve, r=r, pageLength=pageLength)

		@app.route('/r/<int:r>/seen', methods=['POST'])
		def seen(r):
		if not r:
		r = 0
		seenlist=request.form.get('list').split(",")
		# retrieving data
		cve = filter_logic(blacklist, whitelist, unlisted, timeSelect, startDate, endDate,
		timeTypeSelect, cvssSelect, cvss, rejectedSelect, pageLength, r)
		if current_user.is_authenticated():
		col = db.mgmt_seen
		col.update({"user":current_user.get_id()},{"$addToSet":{"seen_cves":{"$each":seenlist}}})
		settings,cve = getFilterSettingsFromPost()
		return render_template('index.html', settings=settings, cve=cve, r=r, pageLength=pageLength)


		@app.route('/r/<int:r>/unseen', methods=['POST'])
		def unseen(r):
		if not r:
		r = 0
		seenlist=request.form.get('list').split(",")
		# retrieving data
		if current_user.is_authenticated():
		col = db.mgmt_seen
		col.update({"user":current_user.get_id()},{"$pullAll":{"seen_cves":seenlist}})
		settings,cve = getFilterSettingsFromPost()
		return render_template('index.html', settings=settings, cve=cve, r=r, pageLength=pageLength)


		@app.route('/api/cpe2.3/<path:cpe>', methods=['GET'])
		def cpe23(cpe):
		cpe = toStringFormattedCPE(cpe)
		@@ -410,6 +446,11 @@ def apisearch(vendor=None, product=None):
		r.append(cve)
		return (json.dumps(r))

		@app.route('/seenCVEs', methods=['POST'])
		def see():
		cves=request.form.get('blacklistSelect')
		return render_template('index.html', cve=cve)

		@app.route('/cve/<cveid>')
		def cve(cveid):
		cveid = cveid.upper()
		@@ -418,9 +459,15 @@ def cve(cveid):
		if cve is None:
		return render_template('error.html',status={'except':'cve-not-found','info':{'cve':cveid}})
		cve = markCPEs(cve)
		if current_user.is_authenticated():
		l = getSeenCVEs()
		if not cveid in l:
		col=db.mgmt_seen
		col.update({"user":current_user.get_id()},{"$addToSet":{"seen_cves":cveid}})
		return render_template('cve.html', cve=cve)



		@app.route('/browse/<vendor>')
		@app.route('/browse/')
		def browse(vendor=None):

Admin message