Commit 2f5069ef authored by Alexandre Dulaunoy's avatar Alexandre Dulaunoy
Browse files

Merge pull request #102 from adulau/master

Exploit databased added in cve-search
parents 831746ff 850779f2
......@@ -65,7 +65,7 @@ A more detailed documentation can be found in the Documentations folder of the p
Databases and collections
-------------------------
The MongoDB database is called cvedb and there are 10 collections:
The MongoDB database is called cvedb and there are 11 collections:
* cves (Common Vulnerabilities and Exposure items) - source NVD NIST
* cpe (Common Platform Enumeration items) - source NVD NIST
......@@ -75,7 +75,8 @@ The MongoDB database is called cvedb and there are 10 collections:
* ranking (ranking rules per group) - local cve-search
* d2sec (Exploitation reference from D2 Elliot Web Exploitation Framework) - source d2sec.com
* [vFeed](https://github.com/toolswatch/vFeed) (cross-references to CVE ids (e.g. OVAL, OpenVAS, ...)) - source [vFeed](https://github.com/toolswatch/vFeed)
* Microsoft Bulletin (Security Vulnerabilities and Bulletin) - source [Microsoft](http://www.microsoft.com/en-us/download/details.aspx?id=36982)
* ms - (Microsoft Bulletin (Security Vulnerabilities and Bulletin)) - source [Microsoft](http://www.microsoft.com/en-us/download/details.aspx?id=36982)
* exploitdb (Offensive Security - Exploit Database) - source [offensive security](https://github.com/offensive-security/exploit-database)
* info (metadata of each collection like last-modified) - local cve-search
The Redis database has 3 databases:
......
......@@ -23,6 +23,7 @@ Vendor: https://nvd.nist.gov/download/vendorstatements.xml
CAPEC: http://capec.mitre.org/data/xml/capec_v2.6.xml
MSBULLETIN: http://download.microsoft.com/download/6/7/3/673E4349-1CA5-40B9-8879-095C72D5B49D/BulletinSearch.xlsx
Ref: https://cve.mitre.org/data/refs/refmap/allrefmaps.zip
exploitdb: https://github.com/offensive-security/exploit-database/raw/master/files.csv
[Webserver]
Host: 127.0.0.1
Port: 5000
......
......@@ -49,6 +49,7 @@ class Configuration():
'capec': "http://capec.mitre.org/data/xml/capec_v2.6.xml",
'msbulletin': "http://download.microsoft.com/download/6/7/3/673E4349-1CA5-40B9-8879-095C72D5B49D/BulletinSearch.xlsx",
'ref': "https://cve.mitre.org/data/refs/refmap/allrefmaps.zip",
'exploitdb': "https://github.com/offensive-security/exploit-database/raw/master/files.csv",
'logging': True, 'logfile': "../log/cve-search.log",
'maxLogSize': '100MB', 'backlog': 5,
'Indexdir': './indexdir',
......@@ -224,6 +225,11 @@ class Configuration():
@classmethod
def getMSBULLETINDict(cls):
return cls.readSetting("Sources", "MSBULLETIN", cls.default['msbulletin'])
@classmethod
def getexploitdbDict(cls):
return cls.readSetting("Sources", "exploitdb", cls.default['exploitdb'])
# Logging
@classmethod
......
......@@ -41,3 +41,4 @@ setIndex('d2sec', 'id')
setIndex('mgmt_whitelist', 'id')
setIndex('mgmt_blacklist', 'id')
setIndex('capec', 'related_weakness')
setIndex('exploitdb', 'id')
#!/usr/bin/env python3
# coding=utf-8
#
# Import exploit database into cve-search
#
# Origin: https://github.com/offensive-security/exploit-database
#
# Software is free software released under the "Modified BSD license"
#
# Copyright (c) 2015 Alexandre Dulaunoy - a@foo.be
import os
import sys
import shutil
runPath = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(runPath, ".."))
from lib.Config import Configuration
import csv
# dictionary
exploitdburl = Configuration.getexploitdbDict()
tmppath = Configuration.getTmpdir()
# connect to db
db = Configuration.getMongoConnection()
exploitdb = db.exploitdb
info = db.info
try:
f = Configuration.getFile(exploitdburl)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(exploitdburl))
i = info.find_one({'db': 'exploitdb'})
if i is not None:
if f.headers['last-modified'] == i['last-modified']:
print("Not modified")
sys.exit(0)
if not os.path.exists(tmppath):
os.mkdir(tmppath)
csvfile = tmppath+'/exploitdb.csv'
with open(csvfile, 'wb') as fp:
shutil.copyfileobj(f, fp)
fp.close()
bulk = exploitdb.initialize_ordered_bulk_op()
with open(csvfile, newline='') as csvtoparse:
exploitcsv = csv.DictReader(csvtoparse, delimiter=',')
for row in exploitcsv:
bulk.find({'id': row['id']}).upsert().update({"$set": {'description': row['description'], 'type': row['type'], 'date': row['date'], 'port': row['port'], 'author': row['author'], 'file': row['file'], 'platform': row['platform'], 'id': row['id']}})
bulk.execute()
# Update last-modified
info.update({'db': 'exploitdb'}, {"$set": {'last-modified': f.headers['last-modified']}}, upsert=True)
......@@ -44,7 +44,10 @@ sources = [{'name': "cves",
{'name': 'ms',
'updater': "python3 " + os.path.join(runPath, "db_mgmt_ms.py")},
{'name': 'redis-nist-ref',
'updater': "python3 " + os.path.join(runPath, "db_mgmt_ref.py")}]
'updater': "python3 " + os.path.join(runPath, "db_mgmt_ref.py")},
{'name': 'exploitdb',
'updater': "python3 " + os.path.join(runPath, "db_mgmt_exploitdb.py")}
]
posts = [{'name': "ensureindex",
'updater': "python3 " + os.path.join(runPath, "db_mgmt_create_index.py")}]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment