Commit 4a6abc50 authored by Alexandre Dulaunoy's avatar Alexandre Dulaunoy Committed by GitHub
Browse files

Merge pull request #211 from adulau/master 

via4cvs and many other fixes
parents 762017f9 87cabfc2

LICENSE

0 → 100644
+16 −0
Original line number Diff line number Diff line 
cve-search is free software and licensed under 3-Clause BSD License - "New BSD License" or "Modified BSD License".



Copyright (c) 2012 Wim Remes - https://github.com/wimremes/

Copyright (c) 2012-2016 Alexandre Dulaunoy - https://github.com/adulau/

Copyright (c) 2015-2016 Pieter-Jan Moreels - https://github.com/pidgeyl/



Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:



1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.



2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.



3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.



THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

bin/db_dump.py

+3 −3
Original line number Diff line number Diff line
@@ -21,16 +21,16 @@ import lib.DatabaseLayer as db 


argParser = argparse.ArgumentParser(description='Dump database in JSON format')

argParser.add_argument('-r', default=False, action='store_true', help='Include ranking value')

argParser.add_argument('-v', default=False, action='store_true', help='Include vfeed map') # TODO change

argParser.add_argument('-v', default=False, action='store_true', help='Include via4 map')

argParser.add_argument('-c', default=False, action='store_true', help='Include CAPEC information')

argParser.add_argument('-l', default=False, type=int, help='Limit output to n elements (default: unlimited)')

args = argParser.parse_args()



rankinglookup = args.r

reflookup = args.v

via4lookup = args.v

capeclookup = args.c



l = cves.last(rankinglookup=rankinglookup, reflookup=reflookup, capeclookup=capeclookup)

l = cves.last(rankinglookup=rankinglookup, via4lookup=via4lookup, capeclookup=capeclookup)



for cveid in db.getCVEIDs(limit=args.l):

    item = l.getcve(cveid=cveid)

bin/search_irc.py

+71 −62
Original line number Diff line number Diff line
@@ -18,20 +18,21 @@ 
# Copyright (c) 2015	 	Pieter-Jan Moreels - pieterjan.moreels@gmail.com



# Imports

import argparse

import irc.bot

import irc.strings

import json

import os

import signal

import ssl

import sys

runPath = os.path.dirname(os.path.realpath(__file__))

sys.path.append(os.path.join(runPath, ".."))



import argparse

import json

# BSON MongoDB include ugly stuff that needs to be processed for standard JSON

from bson    import json_util



import irc.bot

import irc.strings



from lib.Query import lastentries, apigetcve, apibrowse, apisearch

from web.api import API



argParser = argparse.ArgumentParser(description='IRC bot to query cve-search')

argParser.add_argument('-s',    type=str, help='server ip', default='localhost')
@@ -43,14 +44,16 @@ argParser.add_argument('-c', nargs="*", help='channel list', default=['cve-searc 
argParser.add_argument('-t',    type=str, help='trigger prefix', default='.')

argParser.add_argument('-v',    action='store_true', help='channel list', default=['cve-search'])

argParser.add_argument('-m',    type=int, help='maximum query amount', default=20)

argParser.add_argument('--ssl', action='store_true', help='Use SSL')

args = argParser.parse_args()



class IRCBot(irc.bot.SingleServerIRCBot):

  def __init__(self, channel, nickname, server, port, password=None, username=None):

  def __init__(self, channel, nickname, server, port, password=None, username=None, **kwargs):

    if not username:

      username=nickname

    irc.bot.SingleServerIRCBot.__init__(self, [(server, port)], nickname, username)

    irc.bot.SingleServerIRCBot.__init__(self, [(server, port)], nickname, username, **kwargs)

    self.channel = channel

    self.api = API()



  def on_nicknameinuse(self, c, e):

    c.nick(c.get_nickname() + "_")
@@ -74,60 +77,60 @@ class IRCBot(irc.bot.SingleServerIRCBot): 
    return



  def reply(self, e, reply):

    c = self.connection

    if e.target == c.nickname:

    if type(reply) in [dict, list]:

      #reply = json.dumps(reply, sort_keys=True, indent=4, default=json_util.default, ensure_ascii=True)

      reply = json.dumps(reply, sort_keys=True, ensure_ascii=True, default=json_util.default)

    else:

      reply = str(reply)



    if e.target == self.connection.nickname:

      target=e.source.nick

    else:

      target=e.target

    list = reply.split('\n')

    for r in list:

      c.privmsg(target, r)

    _list = reply.split('\n')

    chunk_size = 512 - 12 - len(e.target) # 512 - len("PRIVMSG") - len(" :") - CR/LF - target



  def do_command(self, e, cmd):

    words = cmd.split(' ')

    if len(words)>=2:

      cmd=words[0]

      option=words[1]

    else:

      option=None

    _list = [[x[i:i+chunk_size] for i in range(0, len(x), chunk_size)] for x in _list]

    _list = [item for sublist in _list for item in sublist] # flatten list

    for r in _list[:4]:

      self.connection.privmsg(target, r)



    if cmd == "die":

      self.die()

    elif cmd == "last":

      if option is None:

        limit = 10

      else:

        limit = int(option)

  def do_command(self, e, cmd):

    def last(option):

      limit = int(option) if option else 10

      if limit > args.m or limit < 1:

        self.reply(e, "Request not in range 0-%d" % args.m)

      self.reply(e, json.dumps(lastentries(limit=limit), sort_keys=True, indent=4, default=json_util.default))

    elif cmd == "get":

      self.reply(e, self.api.api_last(limit))

    def cve(option):

      if option is None:

        self.reply(e, "A cve-id must be specified")

      self.reply(e, apigetcve(cveid=option))

    elif cmd == "browse":

      self.reply(e, apibrowse(vendor=option))

    elif cmd == "search":

      self.reply(e, apisearch(query=option))

    elif cmd == "cvetweet":

          return "A cve-id must be specified"

      return self.api.api_cve(option)



    if not cmd: pass

    parts  = cmd.split(' ', 1)

    cmd    = parts[0]

    option = parts[1] if len(parts) == 2 else None



    if   cmd == "die":                self.die()

    elif cmd in ["last", "recent"]:   self.reply(e, last(option))

    elif cmd in ["get", "cve"]:       self.reply(e, cve(option))

    elif cmd in ["browse", "vendor"]: self.reply(e, self.api.api_browse(option))

    elif cmd in ["search", "product"]:

      parts = option.split()

      if len(parts) < 2:

        return self.reply(e, "Usage: search <vendor> <product>")

      return self.reply(e, self.api.api_search(parts[0], parts[1]))

    elif cmd in ["cvetweet", "tweet"]:

      text = ""

      if option is None:

        limit = 10

      else:

        limit = int(option)

      if limit > args.m or limit < 1:

        return "Request not in range 0-%d" % args.m

      for t in lastentries(limit=limit):

        text = text + str(t['id']) + " , " + str(t['summary']) + " " + " , ".join(t['references']) + "\n"

      self.reply(e, text)

    elif cmd == "browse":

        self.reply(e, apibrowse(vendor=option))



      cves = []

      if option.lower().startswith("cve-"): cves.append(cve(option))

      else:                                 cves = last(option)

      for t in cves:

        text += str(t['id']) + " , " + str(t['summary']) + " " + " , ".join(t['references']) + "\n"

      return self.reply(e, text)

    else:

      self.reply(e, "Not understood: " + cmd)



import signal



# signal handlers

def sig_handler(sig, frame):

    print('Caught signal: %s\nShutting down' % sig)
@@ -141,6 +144,12 @@ def main(): 
  user = args.u

  chans = args.c

  global bot



  if args.ssl:

    print("using ssl")

    ssl_factory = irc.connection.Factory(wrapper=ssl.wrap_socket)

    bot=IRCBot(chans, nick, server, port, password=password,username=user, connect_factory=ssl_factory)

  else:

    bot=IRCBot(chans, nick, server, port, password=password,username=user)

  signal.signal(signal.SIGTERM, sig_handler)

  signal.signal(signal.SIGINT, sig_handler)

bin/search_xmpp.py

+56 −59
Original line number Diff line number Diff line
@@ -20,20 +20,20 @@ 
# Copyright (c) 2015	 	Pieter-Jan Moreels - pieterjan.moreels@gmail.com



# Imports

import getpass

import json

import logging

import os

import sleekxmpp

import sys

runPath = os.path.dirname(os.path.realpath(__file__))

sys.path.append(os.path.join(runPath, ".."))



import logging

import getpass

from optparse import OptionParser

import sleekxmpp

import json



from lib.Query import lastentries, apigetcve, apibrowse, apisearch

# BSON MongoDB include ugly stuff that needs to be processed for standard JSON

from bson     import json_util

from optparse import OptionParser



from web.api  import API



if sys.version_info < (3, 0):

    from sleekxmpp.util.misc_ops import setdefaultencoding
@@ -41,51 +41,55 @@ if sys.version_info < (3, 0): 
else:

    raw_input = input



runPath = os.path.dirname(os.path.realpath(__file__))



rankinglookup = True



helpmessage = "\nlast <n> cve entries (output: JSON) \n"

helpmessage = helpmessage + "cvetweet <n> cve entries (output: Text) \n"

helpmessage = helpmessage + "browse vendors and products (output: JSON)\n"

helpmessage = helpmessage + "search <vendor>\<product> (output: JSON)\n"

helpmessage = helpmessage + "get <cve-id> (output: JSON)\n"

helpmessage = helpmessage + "For more info about cve-search: http://adulau.github.com/cve-search/"

helpmessage  = "last [<n>]                - last n cve entries (default: 10) (output: JSON)\n"

helpmessage += "get <cve-id>              - get cve info (output: JSON)\n"

helpmessage += "browse                    - list of vendors (output: JSON)\n"

helpmessage += "browse <vendor>           - list of products of vendor (output: JSON)\n"

helpmessage += "search <vendor> <product> - list of cves for product (output: JSON)\n"

helpmessage += "cvetweet <n>              - summary of <n> last cve entries (output: Text)\n"

helpmessage += "cvetweet <cve-id>         - summary of cve <cve-id> (output: Text) \n\n"

helpmessage += "For more info about cve-search: http://adulau.github.com/cve-search/"



api = API()



def cvesearch(query="last", option=None):

    if query == "last":

        if option is None:

            limit = 10

        else:

            limit = int(option)

    def last(option):

        try:

            limit = int(option) if option else 10

        except:

            return "Please specify the number of CVEs"

        

        if limit > opts.max or limit < 1:

            return "Request not in range 0-%d" % opts.max

        return json.dumps(lastentries(limit=limit), sort_keys=True, indent=4, default=json_util.default)

    elif query == "get":

        return api.api_last(limit)

    def cve(option):

        if option is None:

            return "A cve-id must be specified"

        return apigetcve(opts.api,cveid=option)

    elif query == "browse":

        return apibrowse(opts.api, vendor=option)

    elif query == "search":

        return apisearch(opts.api, query=option)

    elif query == "cvetweet":

        return api.api_cve(option)





    if   query in ["last", "recent"]:    return last(option)

    elif query in ["get", "cve"]:        return cve(option)

    elif query in ["browse", "vendor"]:  return api.api_browse(option)

    elif query in ["search", "product"]:

        parts = option.split()

        if len(parts) < 2:

            return "Usage: search <vendor> <product>"

        return api.api_search(parts[0], parts[1])

    elif query in ["cvetweet", "tweet"]:

        text = ""

        cves = []

        if option.lower().startswith("cve-"): cves.append(cve(option))

        else:                                 cves = last(option)



        if option is None:

            limit = 10

        else:

            limit = int(option)

        if limit > opts.max or limit < 1:

            return "Request not in range 0-%d" % opts.max

        for t in lastentries(limit=limit):

            text = text + str(t['id']) + " , " + str(t['summary']) + " " + " , ".join(t['references']) + "\n"

        for t in cves:

            text += str(t['id']) + " , " + str(t['summary']) + " " + " , ".join(t['references']) + "\n"

        return text

    elif query == "browse":

        return apibrowse(vendor=option)

    else:

        return False

        return helpmessage



class CVEBot(sleekxmpp.ClientXMPP):
@@ -99,6 +103,13 @@ class CVEBot(sleekxmpp.ClientXMPP): 
        self.add_event_handler("message", self.message)

        self.add_event_handler("ssl_invalid_cert", self.ssl_invalid_cert)



    def format_message(self, message):

        if type(message) in [dict, list]:

            message = json.dumps(message, sort_keys=True, indent=4, default=json_util.default)

        else:

            message = str(message)

        return message



    def ssl_invalid_cert(self, cert):

        return
@@ -108,24 +119,11 @@ class CVEBot(sleekxmpp.ClientXMPP): 


    def message(self, msg):

        if msg['type'] in ('chat', 'normal'):

            q = []

            q = (msg['body']).split()

            try:

                option = q[1]

            except IndexError:

                option = None

            if q[0] == "last":

                msg.reply(cvesearch(query="last", option=option)).send()

            elif q[0] == "browse":

                msg.reply(cvesearch(query="browse", option=option)).send()

            elif q[0] == "get":

                msg.reply(cvesearch(query="get", option=option)).send()

            elif q[0] == "cvetweet":

                msg.reply(cvesearch(query="cvetweet", option=option)).send()

            elif q[0] == "search":

                msg.reply(cvesearch(query="search", option=option)).send()

            else:

                msg.reply(helpmessage).send()

            q = (msg['body']).split(' ', 1)

            option = q[1] if len(q) == 2 else None



            reply = cvesearch(query=q[0], option=option)

            msg.reply(self.format_message(reply)).send()



if __name__ == '__main__':

    optp = OptionParser()
@@ -143,7 +141,6 @@ if __name__ == '__main__': 
    optp.add_option("-j", "--jid", dest="jid",

                    help="JID to use")

    optp.add_option('-m', '--max', dest='max', type='int', default=20, help='Maximum elements to return (default: 20)')

    optp.add_option('-a', '--api', dest='api', default='http://127.0.0.1:5000/', help='HTTP API url (default: http://127.0.0.1:5000)')

    optp.add_option("-4", "--ipv4", action='store_true', dest="ipv4",

                    default=False, help="Force IPv4 only")

    optp.add_option("-p", "--password", dest="password",

etc/sources.ini.sample

+6 −11
Original line number Diff line number Diff line
@@ -2,11 +2,6 @@ 
CVE:    https://static.nvd.nist.gov/feeds/xml/cve/

CPE:    https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.2.xml.zip

CWE:    http://cwe.mitre.org/data/xml/cwec_v2.8.xml.zip

d2sec:      http://www.d2sec.com/exploits/elliot.xml

Vendor: https://nvd.nist.gov/download/vendorstatements.xml.gz

CAPEC:  http://capec.mitre.org/data/xml/capec_v2.6.xml

MSBULLETIN: http://download.microsoft.com/download/6/7/3/673E4349-1CA5-40B9-8879-095C72D5B49D/BulletinSearch.xlsx

exploitdb:  https://github.com/offensive-security/exploit-database/raw/master/files.csv

Ref:        https://cve.mitre.org/data/refs/refmap/allrefmaps.zip

RPM:        https://www.redhat.com/security/data/metrics/rpm-to-cve.xml

RHSA:       https://www.redhat.com/security/data/oval/com.redhat.rhsa-all.xml.bz2
 
VIA4:   http://www.cve-search.org/feeds/via4.json

CRA Git | Maintained and supported by SUSTech CRA and CCSE