merge (4d9c613a) · Commits · Zhang Yushan / cve-search

README.md

+3 −2

Original line number	Diff line number	Diff line
		@@ -65,7 +65,7 @@ A more detailed documentation can be found in the Documentations folder of the p
		Databases and collections
		-------------------------

		The MongoDB database is called cvedb and there are 10 collections:
		The MongoDB database is called cvedb and there are 11 collections:

		* cves (Common Vulnerabilities and Exposure items) - source NVD NIST
		* cpe (Common Platform Enumeration items) - source NVD NIST
		@@ -75,7 +75,8 @@ The MongoDB database is called cvedb and there are 10 collections:
		* ranking (ranking rules per group) - local cve-search
		* d2sec (Exploitation reference from D2 Elliot Web Exploitation Framework) - source d2sec.com
		* [vFeed](https://github.com/toolswatch/vFeed) (cross-references to CVE ids (e.g. OVAL, OpenVAS, ...)) - source [vFeed](https://github.com/toolswatch/vFeed)
		* Microsoft Bulletin (Security Vulnerabilities and Bulletin) - source [Microsoft](http://www.microsoft.com/en-us/download/details.aspx?id=36982)
		* ms - (Microsoft Bulletin (Security Vulnerabilities and Bulletin)) - source [Microsoft](http://www.microsoft.com/en-us/download/details.aspx?id=36982)
		* exploitdb (Offensive Security - Exploit Database) - source [offensive security](https://github.com/offensive-security/exploit-database)
		* info (metadata of each collection like last-modified) - local cve-search

		The Redis database has 3 databases:

etc/configuration.ini.sample

+1 −0

Original line number	Diff line number	Diff line
		@@ -23,6 +23,7 @@ Vendor: https://nvd.nist.gov/download/vendorstatements.xml
		CAPEC: http://capec.mitre.org/data/xml/capec_v2.6.xml
		MSBULLETIN: http://download.microsoft.com/download/6/7/3/673E4349-1CA5-40B9-8879-095C72D5B49D/BulletinSearch.xlsx
		Ref: https://cve.mitre.org/data/refs/refmap/allrefmaps.zip
		exploitdb: https://github.com/offensive-security/exploit-database/raw/master/files.csv
		[Webserver]
		Host: 127.0.0.1
		Port: 5000

lib/Config.py

+18 −2

Original line number	Diff line number	Diff line
		@@ -21,6 +21,8 @@ import datetime
		import configparser
		import urllib.parse
		import urllib.request as req
		from io import BytesIO
		import gzip

		class Configuration():
		ConfigParser = configparser.ConfigParser()
		@@ -47,6 +49,7 @@ class Configuration():
		'capec': "http://capec.mitre.org/data/xml/capec_v2.6.xml",
		'msbulletin': "http://download.microsoft.com/download/6/7/3/673E4349-1CA5-40B9-8879-095C72D5B49D/BulletinSearch.xlsx",
		'ref': "https://cve.mitre.org/data/refs/refmap/allrefmaps.zip",
		'exploitdb': "https://github.com/offensive-security/exploit-database/raw/master/files.csv",
		'logging': True, 'logfile': "../log/cve-search.log",
		'maxLogSize': '100MB', 'backlog': 5,
		'Indexdir': './indexdir',
		@@ -222,6 +225,11 @@ class Configuration():
		@classmethod
		def getMSBULLETINDict(cls):
		return cls.readSetting("Sources", "MSBULLETIN", cls.default['msbulletin'])

		@classmethod
		def getexploitdbDict(cls):
		return cls.readSetting("Sources", "exploitdb", cls.default['exploitdb'])

		# Logging

		@classmethod
		@@ -311,11 +319,19 @@ class Configuration():
		return cls.readSetting("Proxy", "http", cls.default['http_proxy'])

		@classmethod
		def getFile(cls, getfile):
		def getFile(cls, getfile, compressed=False):
		if cls.getProxy():
		proxy = req.ProxyHandler({'http': cls.getProxy(), 'https': cls.getProxy()})
		auth = req.HTTPBasicAuthHandler()
		opener = req.build_opener(proxy, auth, req.HTTPHandler)
		req.install_opener(opener)
		if not compressed:
		return req.urlopen(getfile)
		else:
		response = req.urlopen(getfile + '.gz')
		data = None
		if 'gzip' in response.info().get('Content-Type'):
		buf = BytesIO(response.read())
		data = gzip.GzipFile(fileobj=buf)
		return (data, response)

sbin/db_mgmt.py

+6 −6

Original line number	Diff line number	Diff line
		@@ -194,12 +194,12 @@ if __name__ == '__main__':
		# get the 'modified' file
		getfile = file_prefix + file_mod + file_suffix
		try:
		f = Configuration.getFile(Configuration.getCVEDict() + getfile)
		(f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
		i = db.getInfo("cve")
		if i is not None:
		if f.headers['last-modified'] == i['last-modified']:
		if r.headers['last-modified'] == i['last-modified']:
		print("Not modified")
		sys.exit(0)
		db.setColUpdate("cve", f.headers['last-modified'])
		@@ -215,7 +215,7 @@ if __name__ == '__main__':
		# if so, update the entry.
		if x:
		if 'cvss' not in item:
		item['cvss'] = defaultvalue['cvss']
		item['cvss'] = None
		if 'cwe' not in item:
		item['cwe'] = defaultvalue['cwe']
		db.updateCVE(item)
		@@ -224,7 +224,7 @@ if __name__ == '__main__':
		# get the 'recent' file
		getfile = file_prefix + file_rec + file_suffix
		try:
		f = Configuration.getFile(Configuration.getCVEDict() + getfile)
		(f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
		parser = make_parser()
		@@ -239,7 +239,7 @@ if __name__ == '__main__':
		if args.v:
		print("item found : " + item['id'])
		if 'cvss' not in item:
		item['cvss'] = defaultvalue['cvss']
		item['cvss'] = None
		else:
		item['cvss'] = float(item['cvss'])
		if 'cwe' not in item:
		@@ -264,7 +264,7 @@ if __name__ == '__main__':
		parser.setContentHandler(ch)
		getfile = file_prefix + str(x) + file_suffix
		try:
		f = Configuration.getFile(Configuration.getCVEDict() + getfile)
		(f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
		parser.parse(f)

sbin/db_mgmt_create_index.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -38,3 +38,4 @@ setIndex('d2sec', 'id')
		setIndex('mgmt_whitelist', 'id')
		setIndex('mgmt_blacklist', 'id')
		setIndex('capec', 'related_weakness')
		setIndex('exploitdb', 'id')

Admin message