fulltext & plugins with objects. Initial cpe_other

          bulk.find({'id': x.id}).upsert().update({'$set': x.dict()})

      bulk.execute()

  def cpe_alternative_upsert(self, data):

    if type(data) is CPE: data = [data]

    if len(data)>0:

      bulk=self.colCPEOTHER.initialize_unordered_bulk_op()

      for x in data:

        if type(x) is CPE:

          bulk.find({'id': x.id}).upsert().update({'$set': x.dict()})

      bulk.execute()

  # Data retrieval

  def cpe_get(self, id):

    cpe = self.sanitize(self.colCPE.find_one({"id": id}))

  # Info #

  ########

  def _getInfo(self, collection):

    return self.sanitize(self.colINFO.find_one({'db': collection.name}))

    return self.sanitize(self.colINFO.find_one({'db': collection.name})) or {}

  def _setUpdate(self, collection, date):

    self.colINFO.update({"db": collection.name},

  def cpe_getUpdate(self):

    return self._getUpdate(self.colCPE)

  # CPE-Other

  def cpeOther_info(self):

    return self._getInfo(self.colCPEOTHER)

  def cpeOther_setUpdate(self, date):

    self._setUpdate(self.colCPEOTHER, date)

  def cpeOther_getUpdate(self):

    return self._getUpdate(self.colCPEOTHER)

  def cpeOther_setMetadata(self, field, data):

    self.colINFO.update({"db": self.colCPEOTHER.name},

                        {"$set": {field: data}}, upsert=True)

  # CWE

  def cwe_info(self):

    return self._getInfo(self.colCWE)

      raise ValueError()

    self.db.cpe_upsert(cpe)

  def alternative_upsert(self, cpe):

    if not isinstance(cpe, list):

      cpe = [cpe]

    if not all(isinstance(x, CPE) for x in cpe):

      raise ValueError()

    self.db.cpe_alternative_upsert(cpe)

  def get(self, id):

    cpe = self.db.cpe_get(toStringFormattedCPE(id))

    return cpe if cpe else CPE(id)

  def getAll(self):

    return self.db.cpe_getAll()

  def alternative_updated(self, date=None, indexed=None):

    if date:

      self.db.cpeOther_setUpdate(date)

      if indexed:

        self.db.cpeOther_setMetadata('indexed', indexed)

    else:

      data = self.db.cpeOther_info()

      return (data.get('last-modified'), data.get('indexed'))

  def updated(self, date=None):

    if date:

      self.db.cpe_setUpdate(date)

#

# Software is free software released under the "Modified BSD license"

#

# Copyright (c) 2016 	Pieter-Jan Moreels - pieterjan.moreels@gmail.com

# Copyright (c) 2016-2017  Pieter-Jan Moreels - pieterjan.moreels@gmail.com

# Imports

import sys

import importlib

import lib.DatabaseLayer as db

from lib.Config        import Configuration as conf

from lib.Config        import ConfigReader

from lib.DatabaseLayer2 import DatabaseLayer

from lib.Singleton     import Singleton

class PluginManager(metaclass=Singleton):

  def __init__(self):

    self.plugins = {}

    self.db = DatabaseLayer()

  def loadPlugins(self):

    settingsReader = ConfigReader(conf.getPluginsettings())

        # Check if already in result data

        try:

          if not any(item==entry['id'] for entry in result['data']):

            entry=db.getCVE(item)

            entry=self.db.CVE.get(item)

            entry['reason']=collection['n']

            result['data'].append(entry)

        except:

# Software is free software released under the "Modified BSD license"

#

# Copyright (c) 2012-2015  Alexandre Dulaunoy - a@foo.be

# Copyright (c) 2015 		Pieter-Jan Moreels - pieterjan.moreels@gmail.com

# Copyright (c) 2015-2017  Pieter-Jan Moreels - pieterjan.moreels@gmail.com

# Imports

import os

import argparse

from whoosh.index  import create_in, exists_in, open_dir

from whoosh.fields import Schema, TEXT, ID

from lib.Config        import Configuration

from lib.DatabaseLayer2 import DatabaseLayer

from lib.ProgressBar   import progressbar

import lib.CVEs as cves

import lib.DatabaseLayer as db

argParser = argparse.ArgumentParser(description='Fulltext indexer for the MongoDB CVE collection')

argParser.add_argument('-v', action='store_true', default=False, help='Verbose logging')

argParser.add_argument('-l', default=5, help='Number of last entries to index (Default: 5) - 0 to index all documents')

argParser.add_argument('-n', action='store_true', default=False, help='lookup complete cpe (Common Platform Enumeration) name for vulnerable configuration to add in the index')

args = argParser.parse_args()

import lib.CVEs as cves

c = cves.last(namelookup=args.n)

def get_schema_writer():

    indexpath = Configuration.getIndexdir()

from whoosh.index import create_in, exists_in, open_dir

from whoosh.fields import Schema, TEXT, ID

schema = Schema(title=TEXT(stored=True), path=ID(stored=True, unique=True), content=TEXT)

    if not os.path.exists(indexpath):

        os.mkdir(indexpath)

    if not exists_in(indexpath):

        schema = Schema(title=TEXT(stored=True), path=ID(stored=True, unique=True), content=TEXT)

        ix = create_in(indexpath, schema)

    else:

        ix = open_dir(indexpath)

def dumpallcveid(entry=None):

    return db.getCVEID if not entry else db.getCVEIDs(int(entry))

def getcve(cveid=None):

    if cveid is None:

        return False

    return db.getCVE(cveid)

for cveid in progressbar(dumpallcveid(entry=args.l),prefix="Processing"):

    try:

        writer = ix.writer()

        return ix.writer()

    except:

        print ("Index is locked. Another db_fulltext process running?")

        sys.exit(1)

    item = getcve(cveid=cveid)

    title = item['summary'][0:70]

    if args.n:

        for v in item['vulnerable_configuration']:

            cpe = c.getcpe(cpeid=v).strip('\n')

            item['summary'] += " " + cpe

    if args.v:

        print ('Indexing CVE-ID ' + str(cveid) + ' ' + title)

    writer.update_document(title=title, path=cveid, content=item['summary'])

def index(limit=5, cpe_lookup=False, verbose=False):

    if limit == 0: limit = -1

    data = DatabaseLayer().CVE.last(limit=limit)

    for cve in progressbar(data,prefix="Processing"):

        writer = get_schema_writer()

        title  = cve.summary[:70]

        data   = cve.summary

        if cpe_lookup:

            for cpe in cve.vulnerable_configuration:

                data += " " + cpe.title

        if verbose:

            print ('Indexing CVE-ID ' + str(cve.id) + ' ' + title)

        writer.update_document(title=title, path=cve.id, content=data)

        writer.commit()

if __name__ == "__main__":

    argParser = argparse.ArgumentParser(description='Fulltext indexer for the MongoDB CVE collection')

    argParser.add_argument('-v', action='store_true', default=False, help='Verbose logging')

    argParser.add_argument('-l', default=5, type=int, help='Number of last entries to index (Default: 5) - 0 to index all documents')

    argParser.add_argument('-n', action='store_true', default=False, help='lookup complete cpe (Common Platform Enumeration) name for vulnerable configuration to add in the index')

    args = argParser.parse_args()

    index(limit=args.l, cpe_lookup=args.n, verbose=args.v)

import urllib

from lib.DatabaseLayer2 import DatabaseLayer

from lib.ProgressBar   import progressbar

import lib.DatabaseLayer as db

db = DatabaseLayer()

# get dates

icve =  db.getLastModified('cve')

icpeo = db.getLastModified('cpeother')

icve           = db.CVE.updated()

icpeo, indexed = db.CPE.alternative_updated()

# check modification date

date = False

if icve is not None and icpeo is not None:

    # Go check date

    if icve >= icpeo:

if not indexed:

    indexed = 0

if icve and icpeo:

    if icpeo == icve:

        print("Not modified")

        sys.exit(0)

    else:

        date = True

# only get collection of new CVE's

collections = []

if date:

    collections = db.getCVEsNewerThan(icve)

else:

    collections = db.getCVEs()

# check cpes for cves and parse and store missing cpes in cpeother

batch = []

cves = db.CVE.query(skip=indexed, sort=("Modified", "asc"))

print(len(cves))

# skip on empty collections

col=list(collections)

if not col:

if not cves:

    print ("Empty collections, import skipped")

    sys.exit(2)

for item in progressbar(col):

    for cpeentry in item['vulnerable_configuration']:

        checkdup = db.getAlternativeCPE(cpeentry)

        if checkdup and len(checkdup) <= 0:

            entry = db.getCPE(cpeentry)

            if entry and len(entry.count) <= 0:

                title = cpeentry

                title = title[10:]

                title = title.replace(':-:', ' ',10)

                title = title.replace(':', ' ',10)

                title = title.replace('_', ' ',10)

                title = urllib.parse.unquote_plus(title)

                title = title.title()

                batch.append({'id': cpeentry, 'title': title})

if len(batch) != 0:

    db.cpeotherBulkInsert(batch)

unique = set()

for cve in progressbar(cves):

    for cpe in cve.vulnerable_configuration:

        unique.add(cpe.id)

for cpe in unique:

    if db.CPE.get(cpe):

        unique.remove(cpe)

if len(unique) > 0:

    db.CPE.alternative_upsert([CPE(x) for x in unique])

#update database info after successful program-run

db.setColUpdate('cpeother', icve)

db.CVE.updated(icve)

db.CPE.alternative_updated(icve, (indexed+len(cves)))