Merge pull request #99 from adulau/master (f40f8e5b) · Commits · Zhang Yushan / cve-search

doc/talks/brucon2015-cve-search.pdf

0 → 100644

+533 KiB

File added.

No diff preview for this file type.

lib/Config.py

+11 −2

Original line number	Diff line number	Diff line
		@@ -21,6 +21,8 @@ import datetime
		import configparser
		import urllib.parse
		import urllib.request as req
		from io import BytesIO
		import gzip

		class Configuration():
		ConfigParser = configparser.ConfigParser()
		@@ -311,11 +313,18 @@ class Configuration():
		return cls.readSetting("Proxy", "http", cls.default['http_proxy'])

		@classmethod
		def getFile(cls, getfile):
		def getFile(cls, getfile, compressed=False):
		if cls.getProxy():
		proxy = req.ProxyHandler({'http': cls.getProxy(), 'https': cls.getProxy()})
		auth = req.HTTPBasicAuthHandler()
		opener = req.build_opener(proxy, auth, req.HTTPHandler)
		req.install_opener(opener)
		if not compressed:
		return req.urlopen(getfile)
		else:
		response = req.urlopen(getfile + '.gz')
		if 'gzip' in response.info().get('Content-Type'):
		buf = BytesIO(response.read())
		data = gzip.GzipFile(fileobj=buf)
		return (data, response)

sbin/db_mgmt.py

+7 −6

Original line number	Diff line number	Diff line
		@@ -198,15 +198,15 @@ if __name__ == '__main__':
		# get the 'modified' file
		getfile = file_prefix + file_mod + file_suffix
		try:
		f = Configuration.getFile(Configuration.getCVEDict() + getfile)
		(f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
		i = dbLayer.getInfo("cve")
		if i is not None:
		if f.headers['last-modified'] == i['last-modified']:
		if r.headers['last-modified'] == i['last-modified']:
		print("Not modified")
		sys.exit(0)
		dbLayer.setColUpdate("cve", f.headers['last-modified'])
		dbLayer.setColUpdate("cve", r.headers['last-modified'])

		# get your parser on !!
		parser = make_parser()
		@@ -219,7 +219,8 @@ if __name__ == '__main__':
		# if so, update the entry.
		if x:
		if 'cvss' not in item:
		item['cvss'] = defaultvalue['cvss']
		#item['cvss'] = defaultvalue['cvss']
		item['cvss'] = None
		if 'cwe' not in item:
		item['cwe'] = defaultvalue['cwe']
		dbLayer.updateCVE(item)
		@@ -228,7 +229,7 @@ if __name__ == '__main__':
		# get the 'recent' file
		getfile = file_prefix + file_rec + file_suffix
		try:
		f = Configuration.getFile(Configuration.getCVEDict() + getfile)
		(f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
		parser = make_parser()
		@@ -268,7 +269,7 @@ if __name__ == '__main__':
		parser.setContentHandler(ch)
		getfile = file_prefix + str(x) + file_suffix
		try:
		f = Configuration.getFile(Configuration.getCVEDict() + getfile)
		(f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
		parser.parse(f)

sbin/db_mgmt_vendorstatements.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -69,12 +69,12 @@ ch = VendorHandler()
		parser.setContentHandler(ch)
		# check modification date
		try:
		f = Configuration.getFile(vendordict)
		(f, r) = Configuration.getFile(vendordict, compressed = True)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict))
		i = info.find_one({'db': 'vendor'})
		if i is not None:
		if f.headers['last-modified'] == i['last-modified']:
		if r.headers['last-modified'] == i['last-modified']:
		print("Not modified")
		sys.exit(0)
		# parse xml and store in database
		@@ -87,4 +87,4 @@ for statement in progressbar(ch.vendor):
		bulk.execute()

		#update database info after successful program-run
		info.update({'db': 'vendor'}, {"$set": {'last-modified': f.headers['last-modified']}}, upsert=True)
		info.update({'db': 'vendor'}, {"$set": {'last-modified': r.headers['last-modified']}}, upsert=True)

Admin message