fulltext & plugins with objects. Initial cpe_other (4d828e72) · Commits · Zhang Yushan / cve-search

lib/Database.py

+25 −1

Original line number	Original line	Diff line number	Diff line
	@@ -259,6 +259,15 @@ class Database(metaclass=Singleton):
	bulk.find({'id': x.id}).upsert().update({'$set': x.dict()})		bulk.find({'id': x.id}).upsert().update({'$set': x.dict()})
	bulk.execute()		bulk.execute()

			def cpe_alternative_upsert(self, data):
			if type(data) is CPE: data = [data]
			if len(data)>0:
			bulk=self.colCPEOTHER.initialize_unordered_bulk_op()
			for x in data:
			if type(x) is CPE:
			bulk.find({'id': x.id}).upsert().update({'$set': x.dict()})
			bulk.execute()

	# Data retrieval		# Data retrieval
	def cpe_get(self, id):		def cpe_get(self, id):
	cpe = self.sanitize(self.colCPE.find_one({"id": id}))		cpe = self.sanitize(self.colCPE.find_one({"id": id}))
	@@ -326,7 +335,7 @@ class Database(metaclass=Singleton):
	# Info #		# Info #
	########		########
	def _getInfo(self, collection):		def _getInfo(self, collection):
	return self.sanitize(self.colINFO.find_one({'db': collection.name}))		return self.sanitize(self.colINFO.find_one({'db': collection.name})) or {}

	def _setUpdate(self, collection, date):		def _setUpdate(self, collection, date):
	self.colINFO.update({"db": collection.name},		self.colINFO.update({"db": collection.name},
	@@ -356,6 +365,21 @@ class Database(metaclass=Singleton):
	def cpe_getUpdate(self):		def cpe_getUpdate(self):
	return self._getUpdate(self.colCPE)		return self._getUpdate(self.colCPE)

			# CPE-Other
			def cpeOther_info(self):
			return self._getInfo(self.colCPEOTHER)

			def cpeOther_setUpdate(self, date):
			self._setUpdate(self.colCPEOTHER, date)

			def cpeOther_getUpdate(self):
			return self._getUpdate(self.colCPEOTHER)

			def cpeOther_setMetadata(self, field, data):
			self.colINFO.update({"db": self.colCPEOTHER.name},
			{"$set": {field: data}}, upsert=True)


	# CWE		# CWE
	def cwe_info(self):		def cwe_info(self):
	return self._getInfo(self.colCWE)		return self._getInfo(self.colCWE)

lib/DatabaseLayer2.py

+16 −0

Original line number	Original line	Diff line number	Diff line
	@@ -284,6 +284,13 @@ class CPEs:
	raise ValueError()		raise ValueError()
	self.db.cpe_upsert(cpe)		self.db.cpe_upsert(cpe)

			def alternative_upsert(self, cpe):
			if not isinstance(cpe, list):
			cpe = [cpe]
			if not all(isinstance(x, CPE) for x in cpe):
			raise ValueError()
			self.db.cpe_alternative_upsert(cpe)

	def get(self, id):		def get(self, id):
	cpe = self.db.cpe_get(toStringFormattedCPE(id))		cpe = self.db.cpe_get(toStringFormattedCPE(id))
	return cpe if cpe else CPE(id)		return cpe if cpe else CPE(id)
	@@ -294,6 +301,15 @@ class CPEs:
	def getAll(self):		def getAll(self):
	return self.db.cpe_getAll()		return self.db.cpe_getAll()

			def alternative_updated(self, date=None, indexed=None):
			if date:
			self.db.cpeOther_setUpdate(date)
			if indexed:
			self.db.cpeOther_setMetadata('indexed', indexed)
			else:
			data = self.db.cpeOther_info()
			return (data.get('last-modified'), data.get('indexed'))

	def updated(self, date=None):		def updated(self, date=None):
	if date:		if date:
	self.db.cpe_setUpdate(date)		self.db.cpe_setUpdate(date)

lib/PluginManager.py

+7 −6

Original line number	Original line	Diff line number	Diff line
	@@ -5,7 +5,7 @@
	#		#
	# Software is free software released under the "Modified BSD license"		# Software is free software released under the "Modified BSD license"
	#		#
	# Copyright (c) 2016 Pieter-Jan Moreels - pieterjan.moreels@gmail.com		# Copyright (c) 2016-2017 Pieter-Jan Moreels - pieterjan.moreels@gmail.com

	# Imports		# Imports
	import sys		import sys
	@@ -15,14 +15,15 @@ sys.path.append(os.path.join(runPath, ".."))

	import importlib		import importlib

	import lib.DatabaseLayer as db
	from lib.Config import Configuration as conf		from lib.Config import Configuration as conf
	from lib.Config import ConfigReader		from lib.Config import ConfigReader
			from lib.DatabaseLayer2 import DatabaseLayer
	from lib.Singleton import Singleton		from lib.Singleton import Singleton

	class PluginManager(metaclass=Singleton):		class PluginManager(metaclass=Singleton):
	def __init__(self):		def __init__(self):
	self.plugins = {}		self.plugins = {}
			self.db = DatabaseLayer()

	def loadPlugins(self):		def loadPlugins(self):
	settingsReader = ConfigReader(conf.getPluginsettings())		settingsReader = ConfigReader(conf.getPluginsettings())
	@@ -143,7 +144,7 @@ class PluginManager(metaclass=Singleton):
	# Check if already in result data		# Check if already in result data
	try:		try:
	if not any(item==entry['id'] for entry in result['data']):		if not any(item==entry['id'] for entry in result['data']):
	entry=db.getCVE(item)		entry=self.db.CVE.get(item)
	entry['reason']=collection['n']		entry['reason']=collection['n']
	result['data'].append(entry)		result['data'].append(entry)
	except:		except:

sbin/db_fulltext.py

+41 −44

Original line number	Original line	Diff line number	Diff line
	@@ -12,7 +12,7 @@
	# Software is free software released under the "Modified BSD license"		# Software is free software released under the "Modified BSD license"
	#		#
	# Copyright (c) 2012-2015 Alexandre Dulaunoy - a@foo.be		# Copyright (c) 2012-2015 Alexandre Dulaunoy - a@foo.be
	# Copyright (c) 2015 Pieter-Jan Moreels - pieterjan.moreels@gmail.com		# Copyright (c) 2015-2017 Pieter-Jan Moreels - pieterjan.moreels@gmail.com

	# Imports		# Imports
	import os		import os
	@@ -22,55 +22,52 @@ sys.path.append(os.path.join(runPath, ".."))

	import argparse		import argparse

			from whoosh.index import create_in, exists_in, open_dir
			from whoosh.fields import Schema, TEXT, ID

	from lib.Config import Configuration		from lib.Config import Configuration
			from lib.DatabaseLayer2 import DatabaseLayer
	from lib.ProgressBar import progressbar		from lib.ProgressBar import progressbar
	import lib.CVEs as cves
	import lib.DatabaseLayer as db

	argParser = argparse.ArgumentParser(description='Fulltext indexer for the MongoDB CVE collection')		import lib.CVEs as cves
	argParser.add_argument('-v', action='store_true', default=False, help='Verbose logging')
	argParser.add_argument('-l', default=5, help='Number of last entries to index (Default: 5) - 0 to index all documents')
	argParser.add_argument('-n', action='store_true', default=False, help='lookup complete cpe (Common Platform Enumeration) name for vulnerable configuration to add in the index')
	args = argParser.parse_args()

	c = cves.last(namelookup=args.n)

			def get_schema_writer():
	indexpath = Configuration.getIndexdir()		indexpath = Configuration.getIndexdir()

	from whoosh.index import create_in, exists_in, open_dir
	from whoosh.fields import Schema, TEXT, ID

	schema = Schema(title=TEXT(stored=True), path=ID(stored=True, unique=True), content=TEXT)

	if not os.path.exists(indexpath):		if not os.path.exists(indexpath):
	os.mkdir(indexpath)		os.mkdir(indexpath)

	if not exists_in(indexpath):		if not exists_in(indexpath):
			schema = Schema(title=TEXT(stored=True), path=ID(stored=True, unique=True), content=TEXT)
	ix = create_in(indexpath, schema)		ix = create_in(indexpath, schema)
	else:		else:
	ix = open_dir(indexpath)		ix = open_dir(indexpath)
	def dumpallcveid(entry=None):
	return db.getCVEID if not entry else db.getCVEIDs(int(entry))


	def getcve(cveid=None):
	if cveid is None:
	return False
	return db.getCVE(cveid)

	for cveid in progressbar(dumpallcveid(entry=args.l),prefix="Processing"):
	try:		try:
	writer = ix.writer()		return ix.writer()
	except:		except:
	print ("Index is locked. Another db_fulltext process running?")		print ("Index is locked. Another db_fulltext process running?")
	sys.exit(1)		sys.exit(1)
	item = getcve(cveid=cveid)
	title = item['summary'][0:70]		def index(limit=5, cpe_lookup=False, verbose=False):
	if args.n:		if limit == 0: limit = -1
	for v in item['vulnerable_configuration']:		data = DatabaseLayer().CVE.last(limit=limit)
	cpe = c.getcpe(cpeid=v).strip('\n')		for cve in progressbar(data,prefix="Processing"):
	item['summary'] += " " + cpe		writer = get_schema_writer()
	if args.v:		title = cve.summary[:70]
	print ('Indexing CVE-ID ' + str(cveid) + ' ' + title)		data = cve.summary
	writer.update_document(title=title, path=cveid, content=item['summary'])		if cpe_lookup:
			for cpe in cve.vulnerable_configuration:
			data += " " + cpe.title
			if verbose:
			print ('Indexing CVE-ID ' + str(cve.id) + ' ' + title)
			writer.update_document(title=title, path=cve.id, content=data)
	writer.commit()		writer.commit()

			if __name__ == "__main__":
			argParser = argparse.ArgumentParser(description='Fulltext indexer for the MongoDB CVE collection')
			argParser.add_argument('-v', action='store_true', default=False, help='Verbose logging')
			argParser.add_argument('-l', default=5, type=int, help='Number of last entries to index (Default: 5) - 0 to index all documents')
			argParser.add_argument('-n', action='store_true', default=False, help='lookup complete cpe (Common Platform Enumeration) name for vulnerable configuration to add in the index')
			args = argParser.parse_args()

			index(limit=args.l, cpe_lookup=args.n, verbose=args.v)

sbin/db_mgmt_cpe_other_dictionary.py

+24 −40

Original line number	Original line	Diff line number	Diff line
	@@ -33,55 +33,39 @@ sys.path.append(os.path.join(runPath, ".."))

	import urllib		import urllib

			from lib.DatabaseLayer2 import DatabaseLayer
	from lib.ProgressBar import progressbar		from lib.ProgressBar import progressbar
	import lib.DatabaseLayer as db

			db = DatabaseLayer()
	# get dates		# get dates
	icve = db.getLastModified('cve')		icve = db.CVE.updated()
	icpeo = db.getLastModified('cpeother')		icpeo, indexed = db.CPE.alternative_updated()

	# check modification date		if not indexed:
	date = False		indexed = 0
	if icve is not None and icpeo is not None:
	# Go check date		if icve and icpeo:
	if icve >= icpeo:		if icpeo == icve:
	print("Not modified")		print("Not modified")
	sys.exit(0)		sys.exit(0)
	else:
	date = True

	# only get collection of new CVE's		cves = db.CVE.query(skip=indexed, sort=("Modified", "asc"))
	collections = []		print(len(cves))
	if date:
	collections = db.getCVEsNewerThan(icve)
	else:
	collections = db.getCVEs()
	# check cpes for cves and parse and store missing cpes in cpeother
	batch = []

	# skip on empty collections		if not cves:
	col=list(collections)
	if not col:
	print ("Empty collections, import skipped")		print ("Empty collections, import skipped")
	sys.exit(2)		sys.exit(2)

	for item in progressbar(col):		unique = set()
	for cpeentry in item['vulnerable_configuration']:		for cve in progressbar(cves):
	checkdup = db.getAlternativeCPE(cpeentry)		for cpe in cve.vulnerable_configuration:
	if checkdup and len(checkdup) <= 0:		unique.add(cpe.id)
	entry = db.getCPE(cpeentry)		for cpe in unique:
	if entry and len(entry.count) <= 0:		if db.CPE.get(cpe):
	title = cpeentry		unique.remove(cpe)
	title = title[10:]		if len(unique) > 0:
	title = title.replace(':-:', ' ',10)		db.CPE.alternative_upsert([CPE(x) for x in unique])
	title = title.replace(':', ' ',10)
	title = title.replace('_', ' ',10)
	title = urllib.parse.unquote_plus(title)

	title = title.title()
	batch.append({'id': cpeentry, 'title': title})
	if len(batch) != 0:
	db.cpeotherBulkInsert(batch)

	#update database info after successful program-run		#update database info after successful program-run
	db.setColUpdate('cpeother', icve)		db.CVE.updated(icve)
			db.CPE.alternative_updated(icve, (indexed+len(cves)))

Admin message