{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib, json, shelve, gzip\n",
    "file = \"https://nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-2019.json.gz\"\n",
    "print('Fetching nvdcve-1.0-2019.json.gz')\n",
    "test = urllib.request.urlopen(file)\n",
    "data = test.read()\n",
    "print('Storing Gzipped File')\n",
    "with open('nvdcve-1.0-2019.json.gz', 'wb') as f:\n",
    "    f.write(data)\n",
    "with gzip.open('./nvdcve-1.0-2019.json.gz', 'rb') as f:\n",
    "    file_content = f.read()\n",
    "print('Loading JSON Content')\n",
    "cve_data = json.loads(file_content)\n",
    "print('{} records'.format(len(cve_data['CVE_Items'])))\n",
    "raw_records = [x['cve']['CVE_data_meta']['ID'] for x in cve_data['CVE_Items']]\n",
    "print('{} unique records'.format(len(set(x['cve']['CVE_data_meta']['ID'] for x in cve_data['CVE_Items']))))\n",
    "print('Creating Shelve: cve_2019.shelf')\n",
    "db = shelve.open('cve_2019.shelf')\n",
    "print('Assembling Big Dictionary of 2019 Data in shelve')\n",
    "CVES = cve_data['CVE_Items']\n",
    "for item in CVES:\n",
    "    #print(item['cve']['CVE_data_meta']['ID'])\n",
    "    db[item['cve']['CVE_data_meta']['ID']] = item['cve']\n",
    "print('shelve reports {} unique records'.format(len(db.keys())))\n",
    "print('Extracting data by keys from shelve')\n",
    "results = []\n",
    "for x in raw_records:\n",
    "    results.append(db[x])\n",
    "print('{} extracted records'.format(len(results)))\n",
    "print('Number of missing records {}'.format(len([x for x in raw_records if x not in db.keys()])))\n",
    "if len(set(x['cve']['CVE_data_meta']['ID'] for x in cve_data['CVE_Items'])) > len(db.keys()):\n",
    "    print('Shelve is on strike.')\n",
    "else:\n",
    "    print('data match')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "mkdir ./test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp ./pbr37241_Jesse_Bacon.py ./test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/jbacon/dev_space/gitlab/CVE_Alert/proof/test\n"
     ]
    }
   ],
   "source": [
    "cd ./test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pbr37241_Jesse_Bacon.py\r\n"
     ]
    }
   ],
   "source": [
    "ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching nvdcve-1.0-2019.json.gz\n",
      "Storing Gzipped File\n",
      "Loading JSON Content\n",
      "4275 records\n",
      "4275 unique records\n",
      "Creating Shelve: cve_2019.shelf\n",
      "Assembling Big Dictionary of 2019 Data in shelve\n",
      "shelve reports 3928 unique records\n",
      "Extracting data by keys from shelve\n",
      "4275 extracted records\n",
      "Number of missing records 0\n",
      "Shelve is on strike.\n"
     ]
    }
   ],
   "source": [
    "%run ./pbr37241_Jesse_Bacon.py"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}