more database layer abstraction (f59302d0) · Commits · Zhang Yushan / cve-search

sbin/db_fulltext.py

+4 −14

Original line number	Diff line number	Diff line
		@@ -22,13 +22,10 @@ sys.path.append(os.path.join(runPath, ".."))

		import argparse

		import lib.CVEs as cves
		from lib.Config import Configuration
		from lib.ProgressBar import progressbar

		# connect to db
		db = Configuration.getMongoConnection()
		collection = db.cves
		import lib.CVEs as cves
		import lib.DatabaseLayer as db

		argParser = argparse.ArgumentParser(description='Fulltext indexer for the MongoDB CVE collection')
		argParser.add_argument('-v', action='store_true', default=False, help='Verbose logging')
		@@ -53,20 +50,13 @@ if not exists_in(indexpath):
		else:
		ix = open_dir(indexpath)
		def dumpallcveid(entry=None):
		cveid = []
		if entry is None:
		for x in collection.find({}).sort('_id', 1):
		cveid.append(x['id'])
		else:
		for x in collection.find({}).sort("Modified", -1).limit(int(entry)):
		cveid.append(x['id'])
		return cveid
		return db.getCVEID if not entry else db.getCVEIDs(int(entry))


		def getcve(cveid=None):
		if cveid is None:
		return False
		return collection.find_one({'id': cveid})
		return db.getCVE(cveid)

		for cveid in progressbar(dumpallcveid(entry=args.l),prefix="Processing"):
		try:

+3 −4

Original line number	Diff line number	Diff line
		@@ -159,7 +159,6 @@ capecurl = Configuration.getCAPECDict()
		# connect to db
		db = Configuration.getMongoConnection()
		capec = db.capec
		info = db.info

		# make parser
		parser = make_parser()
		@@ -170,9 +169,9 @@ try:
		f = Configuration.getFile(capecurl)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(capecurl))
		i = info.find_one({'db': 'capec'})
		i = dbLayer.getLastModified('capec')
		if i is not None:
		if f.headers['last-modified'] == i['last-modified']:
		if f.headers['last-modified'] == i:
		print("Not modified")
		sys.exit(0)
		# parse xml and store in database
		@@ -183,4 +182,4 @@ for attack in progressbar(ch.capec):
		bulk.execute()

		#update database info after successful program-run
		info.update({'db': 'capec'}, {"$set": {'last-modified': f.headers['last-modified']}}, upsert=True)
		dbLayer.setColUpdate('capec', f.headers['last-modified'])

+4 −3

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@ runPath = os.path.dirname(os.path.realpath(__file__))
		sys.path.append(os.path.join(runPath, ".."))

		from lib.Config import Configuration
		import lib.DatabaseLayer as dbLayer

		import csv

		@@ -33,9 +34,9 @@ try:
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(exploitdburl))

		i = info.find_one({'db': 'exploitdb'})
		i = dbLayer.getLastModified('exploitdb')
		if i is not None:
		if f.headers['last-modified'] == i['last-modified']:
		if f.headers['last-modified'] == i:
		print("Not modified")
		sys.exit(0)

		@@ -57,4 +58,4 @@ with open(csvfile, newline='') as csvtoparse:
		bulk.execute()

		# Update last-modified
		info.update({'db': 'exploitdb'}, {"$set": {'last-modified': f.headers['last-modified']}}, upsert=True)
		dbLayer.setColUpdate('exploitdb', f.headers['last-modified'])

+5 −4

Original line number	Diff line number	Diff line
		@@ -25,6 +25,8 @@ import argparse
		from lib.ProgressBar import progressbar
		from lib.Config import Configuration

		import lib.DatabaseLayer as dbLayer

		argparser = argparse.ArgumentParser(description='populate/update vendor statement database')
		argparser.add_argument('-v', action='store_true', help='verbose output')
		args = argparser.parse_args()
		@@ -61,7 +63,6 @@ vendordict = Configuration.getVendorDict()
		# connect to db
		db = Configuration.getMongoConnection()
		vendor = db.vendor
		info = db.info

		# make parser
		parser = make_parser()
		@@ -72,9 +73,9 @@ try:
		(f, r) = Configuration.getFile(vendordict, compressed = True)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict))
		i = info.find_one({'db': 'vendor'})
		i = dbLayer.getLastModified('vendor')
		if i is not None:
		if r.headers['last-modified'] == i['last-modified']:
		if r.headers['last-modified'] == i:
		print("Not modified")
		sys.exit(0)
		# parse xml and store in database
		@@ -87,4 +88,4 @@ for statement in progressbar(ch.vendor):
		bulk.execute()

		#update database info after successful program-run
		info.update({'db': 'vendor'}, {"$set": {'last-modified': r.headers['last-modified']}}, upsert=True)
		dbLayer.setColUpdate('vendor', r.headers['last-modified'])