Commit 18247fd5 authored by Alexandre Dulaunoy's avatar Alexandre Dulaunoy Committed by GitHub
Browse files

Merge pull request #170 from adulau/master

Many fixes
parents 1fe9fa21 46cca817
......@@ -17,7 +17,7 @@ import re
from optparse import OptionParser
from lib.Query import lastentries, apigetcve, apibrowse, apisearch
from lib.Query import apigetcve
optp = OptionParser()
optp.add_option('-c', '--cve', dest='cve', default='CVE-2015-0001', help='CVE id to convert')
......
......@@ -37,4 +37,8 @@ for cveid in db.getCVEIDs(limit=args.l):
if 'cvss' in item:
if type(item['cvss']) == str:
item['cvss'] = float(item['cvss'])
print (json.dumps(item, sort_keys=True, default=json_util.default))
date_fields = ['cvss-time', 'Modified', 'Published']
for field in date_fields:
if field in item:
item[field] = str(item[field])
print(json.dumps(item, sort_keys=True, default=json_util.default))
......@@ -69,7 +69,7 @@ for x in cvelist.get(limit=last):
print ("<tr class=\"alt\">")
print ("<td>" + str(x['id']) + " - " + x['summary'][:90] + "...</td>")
print ("</tr>")
print ("<tr><td>CVSS: " + str(x['cvss']) + " Published: " + x['Published'] + "</td></tr>")
print ("<tr><td>CVSS: " + str(x['cvss']) + " Published: " + str(x['Published']) + "</td></tr>")
print ("<tr>")
print ("<td> Summary: " + x['summary'] + "</td>")
print ("</tr>")
......
......@@ -94,9 +94,13 @@ if sLatest:
sorttype = -1
def printCVE(item):
def printCVE(item, indent=None):
date_fields = ['cvss-time', 'Modified', 'Published']
for field in date_fields:
if field in item:
item[field] = str(item[field])
if not namelookup and not rankinglookup and not capeclookup:
print(json.dumps(item, sort_keys=True, default=json_util.default))
print(json.dumps(item, sort_keys=True, default=json_util.default, indent=indent))
else:
if "vulnerable_configuration" in item:
vulconf = []
......@@ -115,7 +119,7 @@ def printCVE(item):
if "cwe" in item and capeclookup:
if item['cwe'].lower() != 'unknown':
item['capec'] = cves.getcapec(cweid=(item['cwe'].split('-')[1]))
print(json.dumps(item, sort_keys=True, default=json_util.default))
print(json.dumps(item, sort_keys=True, default=json_util.default, indent=indent))
if cveSearch:
for cveid in db.getCVEs(cves=cveSearch):
......@@ -126,7 +130,7 @@ if cveSearch:
if vFreeSearch:
try:
for item in db.getFreeText(vFreeSearch):
print(item)
printCVE(item, indent=2)
except:
sys.exit("Free text search not enabled on the database!")
sys.exit(0)
......@@ -150,11 +154,11 @@ if vSearch:
nl = " ".join(item['vulnerable_configuration'])
csvoutput = csv.writer(sys.stdout, delimiter='|', quotechar='|', quoting=csv.QUOTE_MINIMAL)
if not namelookup:
csvoutput.writerow([item['id'], item['Published'], item['cvss'], item['summary'], refs])
csvoutput.writerow([item['id'], str(item['Published']), item['cvss'], item['summary'], refs])
else:
csvoutput.writerow([item['id'], item['Published'], item['cvss'], item['summary'], refs, nl])
csvoutput.writerow([item['id'], str(item['Published']), item['cvss'], item['summary'], refs, nl])
elif htmlOutput:
print("<h2>" + item['id'] + "<br></h2>CVSS score: " + str(item['cvss']) + "<br>" + "<b>" + item['Published'] + "<b><br>" + item['summary'] + "<br>")
print("<h2>" + item['id'] + "<br></h2>CVSS score: " + str(item['cvss']) + "<br>" + "<b>" + str(item['Published']) + "<b><br>" + item['summary'] + "<br>")
print("References:<br>")
for entry in item['references']:
print(entry + "<br>")
......@@ -167,7 +171,7 @@ if vSearch:
c = SubElement(r, 'id')
c.text = item['id']
c = SubElement(r, 'Published')
c.text = item['Published']
c.text = str(item['Published'])
c = SubElement(r, 'cvss')
c.text = str(item['cvss'])
c = SubElement(r, 'summary')
......@@ -182,7 +186,7 @@ if vSearch:
print(item['id'])
else:
print("CVE\t: " + item['id'])
print("DATE\t: " + item['Published'])
print("DATE\t: " + str(item['Published']))
print("CVSS\t: " + str(item['cvss']))
print(item['summary'])
print("\nReferences:")
......
File mode changed from 100644 to 100755
File mode changed from 100644 to 100755
......@@ -89,7 +89,7 @@ def bulkvFeedUpdate(dbpath, vfeedmap):
else:
icveid = names.index("cveid")
except Exception as ex:
sys.exit('Exeption in %s: %s' % (vmap, ex))
print('Exeption in %s: %s' % (vmap, ex))
continue
mapArray={}
for i in range(0,len(r)):
......
......@@ -18,13 +18,11 @@ import importlib
import lib.DatabaseLayer as db
from lib.Config import Configuration as conf
from lib.Config import ConfigReader
from lib.Plugins import Plugin, WebPlugin
from flask.ext.login import current_user
class PluginManager():
def __init__(self):
self.plugins = {}
def loadPlugins(self):
settingsReader = ConfigReader(conf.getPluginsettings())
if not os.path.exists(conf.getPluginLoadSettings()):
......@@ -129,23 +127,27 @@ class PluginManager():
print("[!] -> %s"%e)
return cveInfo
def getSearchResults(self, text):
def getSearchResults(self, text, **args):
result = {'data':[]}
results = []
# Get all data
for plugin in self.plugins.values():
data = plugin.search(text)
data = plugin.search(text, **args)
# Validate format
if type(data) == dict: data = [data]
if type(data) == list and all([(type(x) == dict and 'n' in x and 'd' in x) for x in data]):
results.extend(data)
# Sort through data
for collection in results:
for item in collection['d']:
# Check if already in result data
if not any(item==entry['id'] for entry in result['data']):
entry=db.getCVE(item)
entry['reason']=collection['n']
result['data'].append(entry)
try:
if not any(item==entry['id'] for entry in result['data']):
entry=db.getCVE(item)
entry['reason']=collection['n']
result['data'].append(entry)
except:
pass
return result
# Actions
......@@ -176,7 +178,7 @@ class PluginManager():
return ("error.html", {'status': {'except': 'plugin-not-webplugin'}})
return ("error.html", {'status': {'except': 'plugin-not-loaded'}})
def openSubpage(self, subpage, **args):
def openSubpage(self, name, subpage, **args):
if name.strip() in self.plugins.keys(): # Check if plugin exists
if self.plugins[name].isWebPlugin(): # Check if plugin is web plugin
pageInfo = self.plugins[name].getSubpage(subpage, **args)
......
......@@ -28,7 +28,7 @@ class Plugin():
def loadSettings(self, reader): pass
def onDatabaseUpdate(self): pass
# To override with returns
def search(self, text): pass
def search(self, text, **args): pass
class WebPlugin(Plugin):
......
......@@ -10,7 +10,6 @@
# Imports
from dateutil import tz
import dateutil.parser
import time
import re
# Note of warning: CPEs like cpe:/o:microsoft:windows_8:-:-:x64 are given to us by Mitre
......@@ -61,8 +60,8 @@ def impactScore(cve):
I=((cve['impact'])['integrity']).upper()
A=((cve['impact'])['availability']).upper()
res = 10.41*(1-(1-score[C])*(1-score[I])*(1-score[A]))
return 10.0 if res > 10.0 else res
except Exception as ex:
return 10.0 if res > 10.0 else res
except:
return '-'
def exploitabilityScore(cve):
......@@ -100,28 +99,6 @@ def vFeedName(string):
string=string.replace('cve_','')
return string.title()
def convertDateToDBFormat(string):
result = None
try:
result = time.strptime(string, "%d-%m-%Y")
except:
pass
try:
result = time.strptime(string, "%d-%m-%y")
except:
pass
try:
result = time.strptime(string, "%d/%m/%Y")
except:
pass
try:
result = time.strptime(string, "%d/%m/%y")
except:
pass
if result is not None:
result = time.strftime('%Y-%m-%d', result)
return result
def mergeSearchResults(database, plugins):
if 'errors' in database:
results = {'data':[], 'errors':database['errors']}
......
......@@ -15,7 +15,7 @@
import os
runPath = os.path.dirname(os.path.realpath(__file__))
from flask.ext.login import UserMixin
from flask_login import UserMixin
from lib.Config import Configuration
import lib.DatabaseLayer as db
......@@ -35,7 +35,7 @@ class User(UserMixin):
USERS = {}
for user in db.getUsers():
USERS[user['username']] = user['password']
if not id in USERS:
raise UserNotFoundError()
self.id = id
......
......@@ -16,7 +16,6 @@ runPath = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(runPath, ".."))
import json
import re
from lib.Toolkit import toStringFormattedCPE
import lib.DatabaseLayer as db
......
......@@ -16,7 +16,6 @@ sys.path.append(os.path.join(runPath, ".."))
import argparse
from lib.Config import Configuration
from lib.cpelist import CPEList
# parse command line arguments
......
......@@ -19,6 +19,8 @@ import datetime
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
from dateutil.parser import parse as parse_datetime
from lib.ProgressBar import progressbar
from lib.Toolkit import toStringFormattedCPE
from lib.Config import Configuration
......@@ -173,16 +175,16 @@ class CVEHandler(ContentHandler):
self.cves[-1]['impact']['availability'] = self.impacta
if name == 'cvss:generated-on-datetime':
self.inCVSSgenElem = 0
self.cves[-1]['cvss-time'] = self.cvssgen
self.cves[-1]['cvss-time'] = parse_datetime(self.cvssgen, ignoretz=True)
if name == 'vuln:summary':
self.inSUMMElem = 0
self.cves[-1]['summary'] = self.SUMM
if name == 'vuln:published-datetime':
self.inDTElem = 0
self.cves[-1]['Published'] = self.DT
self.cves[-1]['Published'] = parse_datetime(self.DT, ignoretz=True)
if name == 'vuln:last-modified-datetime':
self.inPUBElem = 0
self.cves[-1]['Modified'] = self.PUB
self.cves[-1]['Modified'] = parse_datetime(self.PUB, ignoretz=True)
if __name__ == '__main__':
parser = make_parser()
......@@ -197,11 +199,12 @@ if __name__ == '__main__':
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile))
i = db.getInfo("cve")
last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True)
if i is not None:
if r.headers['last-modified'] == i['last-modified']:
if last_modified == i['last-modified']:
print("Not modified")
sys.exit(0)
db.setColUpdate("cve", r.headers['last-modified'])
db.setColUpdate("cve", last_modified)
# get your parser on !!
parser = make_parser()
......@@ -275,8 +278,7 @@ if __name__ == '__main__':
item['cvss'] = float(item['cvss'])
# check if year is not cve-free
if len(ch.cves) != 0:
print("Importing CVEs for year " + str(x))
ret = db.insertCVE(ch.cves)
if ret:
print ("Year " + str(x) + " imported.")
else:
print ("Year " + str(x) + " has no CVE's.")
......@@ -23,7 +23,6 @@ import argparse
import getpass
from passlib.hash import pbkdf2_sha256
from lib.Config import Configuration
import lib.DatabaseLayer as dbLayer
# args
......
......@@ -14,6 +14,8 @@ sys.path.append(os.path.join(runPath, ".."))
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
from dateutil.parser import parse as parse_datetime
from lib.ProgressBar import progressbar
from lib.Config import Configuration
import lib.DatabaseLayer as db
......@@ -167,8 +169,9 @@ try:
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(capecurl))
i = db.getLastModified('capec')
last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True)
if i is not None:
if f.headers['last-modified'] == i:
if last_modified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
......@@ -179,4 +182,4 @@ for attack in progressbar(ch.capec):
db.bulkUpdate("capec", attacks)
#update database info after successful program-run
db.setColUpdate('capec', f.headers['last-modified'])
db.setColUpdate('capec', last_modified)
......@@ -25,6 +25,8 @@ sys.path.append(os.path.join(runPath, ".."))
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
from dateutil.parser import parse as parse_datetime
from lib.ProgressBar import progressbar
from lib.Toolkit import toStringFormattedCPE
from lib.Config import Configuration
......@@ -81,8 +83,9 @@ try:
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cpedict))
i = db.getLastModified('cpe')
last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True)
if i is not None:
if f.headers['last-modified'] == i:
if last_modified == i:
print("Not modified")
sys.exit(0)
# parse xml and store in database
......@@ -97,4 +100,4 @@ for x in progressbar(ch.cpe):
db.bulkUpdate("cpe", cpeList)
#update database info after successful program-run
db.setColUpdate('cpe', f.headers['last-modified'])
db.setColUpdate('cpe', last_modified)
......@@ -34,7 +34,6 @@ sys.path.append(os.path.join(runPath, ".."))
import urllib
from lib.ProgressBar import progressbar
from lib.Config import Configuration
import lib.DatabaseLayer as db
# get dates
......
......@@ -16,7 +16,6 @@ sys.path.append(os.path.join(runPath, ".."))
from pymongo import TEXT
from lib.Config import Configuration
import lib.DatabaseLayer as dbLayer
def setIndex(col, field):
......
......@@ -25,6 +25,8 @@ import sys
runPath = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(runPath, ".."))
from dateutil.parser import parse as parse_datetime
from xml.sax import make_parser
from xml.sax.handler import ContentHandler
import argparse
......@@ -82,7 +84,7 @@ try:
f = Configuration.getFile(cwedict)
except:
sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cwedict))
lastmodified = f.headers['last-modified']
lastmodified = parse_datetime(f.headers['last-modified'], ignoretz=True)
i = db.getLastModified('cwe')
if i is not None:
if lastmodified == i:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment