Merge pull request #102 from adulau/master (2f5069ef) · Commits · Zhang Yushan / cve-search

README.md

+3 −2

Original line number	Diff line number	Diff line
		@@ -65,7 +65,7 @@ A more detailed documentation can be found in the Documentations folder of the p
		Databases and collections
		-------------------------

		The MongoDB database is called cvedb and there are 10 collections:
		The MongoDB database is called cvedb and there are 11 collections:

		* cves (Common Vulnerabilities and Exposure items) - source NVD NIST
		* cpe (Common Platform Enumeration items) - source NVD NIST
		@@ -75,7 +75,8 @@ The MongoDB database is called cvedb and there are 10 collections:
		* ranking (ranking rules per group) - local cve-search
		* d2sec (Exploitation reference from D2 Elliot Web Exploitation Framework) - source d2sec.com
		* [vFeed](https://github.com/toolswatch/vFeed) (cross-references to CVE ids (e.g. OVAL, OpenVAS, ...)) - source [vFeed](https://github.com/toolswatch/vFeed)
		* Microsoft Bulletin (Security Vulnerabilities and Bulletin) - source [Microsoft](http://www.microsoft.com/en-us/download/details.aspx?id=36982)
		* ms - (Microsoft Bulletin (Security Vulnerabilities and Bulletin)) - source [Microsoft](http://www.microsoft.com/en-us/download/details.aspx?id=36982)
		* exploitdb (Offensive Security - Exploit Database) - source [offensive security](https://github.com/offensive-security/exploit-database)
		* info (metadata of each collection like last-modified) - local cve-search

		The Redis database has 3 databases:

etc/configuration.ini.sample

+1 −0

Original line number	Diff line number	Diff line
		@@ -23,6 +23,7 @@ Vendor: https://nvd.nist.gov/download/vendorstatements.xml
		CAPEC: http://capec.mitre.org/data/xml/capec_v2.6.xml
		MSBULLETIN: http://download.microsoft.com/download/6/7/3/673E4349-1CA5-40B9-8879-095C72D5B49D/BulletinSearch.xlsx
		Ref: https://cve.mitre.org/data/refs/refmap/allrefmaps.zip
		exploitdb: https://github.com/offensive-security/exploit-database/raw/master/files.csv
		[Webserver]
		Host: 127.0.0.1
		Port: 5000

lib/Config.py

+6 −0

Original line number	Diff line number	Diff line
		@@ -49,6 +49,7 @@ class Configuration():
		'capec': "http://capec.mitre.org/data/xml/capec_v2.6.xml",
		'msbulletin': "http://download.microsoft.com/download/6/7/3/673E4349-1CA5-40B9-8879-095C72D5B49D/BulletinSearch.xlsx",
		'ref': "https://cve.mitre.org/data/refs/refmap/allrefmaps.zip",
		'exploitdb': "https://github.com/offensive-security/exploit-database/raw/master/files.csv",
		'logging': True, 'logfile': "../log/cve-search.log",
		'maxLogSize': '100MB', 'backlog': 5,
		'Indexdir': './indexdir',
		@@ -224,6 +225,11 @@ class Configuration():
		@classmethod
		def getMSBULLETINDict(cls):
		return cls.readSetting("Sources", "MSBULLETIN", cls.default['msbulletin'])

		@classmethod
		def getexploitdbDict(cls):
		return cls.readSetting("Sources", "exploitdb", cls.default['exploitdb'])

		# Logging

		@classmethod

sbin/db_mgmt_create_index.py

+1 −0

Original line number	Diff line number	Diff line
		@@ -41,3 +41,4 @@ setIndex('d2sec', 'id')
		setIndex('mgmt_whitelist', 'id')
		setIndex('mgmt_blacklist', 'id')
		setIndex('capec', 'related_weakness')
		setIndex('exploitdb', 'id')

sbin/db_mgmt_exploitdb.py

0 → 100644

+60 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3
		# coding=utf-8
		#
		# Import exploit database into cve-search
		#
		# Origin: https://github.com/offensive-security/exploit-database
		#
		# Software is free software released under the "Modified BSD license"
		#
		# Copyright (c) 2015 Alexandre Dulaunoy - a@foo.be

		import os
		import sys
		import shutil
		runPath = os.path.dirname(os.path.realpath(__file__))
		sys.path.append(os.path.join(runPath, ".."))

		from lib.Config import Configuration

		import csv

		# dictionary
		exploitdburl = Configuration.getexploitdbDict()
		tmppath = Configuration.getTmpdir()

		# connect to db
		db = Configuration.getMongoConnection()
		exploitdb = db.exploitdb
		info = db.info

		try:
		f = Configuration.getFile(exploitdburl)
		except:
		sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(exploitdburl))

		i = info.find_one({'db': 'exploitdb'})
		if i is not None:
		if f.headers['last-modified'] == i['last-modified']:
		print("Not modified")
		sys.exit(0)

		if not os.path.exists(tmppath):
		os.mkdir(tmppath)

		csvfile = tmppath+'/exploitdb.csv'
		with open(csvfile, 'wb') as fp:
		shutil.copyfileobj(f, fp)
		fp.close()

		bulk = exploitdb.initialize_ordered_bulk_op()

		with open(csvfile, newline='') as csvtoparse:
		exploitcsv = csv.DictReader(csvtoparse, delimiter=',')
		for row in exploitcsv:
		bulk.find({'id': row['id']}).upsert().update({"$set": {'description': row['description'], 'type': row['type'], 'date': row['date'], 'port': row['port'], 'author': row['author'], 'file': row['file'], 'platform': row['platform'], 'id': row['id']}})

		bulk.execute()

		# Update last-modified
		info.update({'db': 'exploitdb'}, {"$set": {'last-modified': f.headers['last-modified']}}, upsert=True)

Admin message