Parse HTML table data to JSON and save to text file in Python 2.7

Question

I'm trying to extract the data on the crime rate across states from this webpage, link to web page http://www.disastercenter.com/crime/uscrime.htm

I am able to get this into text file. But I would like to get the response in Json format. How can I do this in python.

Here is my code:

import urllib        
import re     

from bs4 import BeautifulSoup    
link = "http://www.disastercenter.com/crime/uscrime.htm"    
f = urllib.urlopen(link)    
myfile = f.read()    
soup = BeautifulSoup(myfile)    
soup1=soup.find('table', width="100%")    
soup3=str(soup1)    
result = re.sub("<.*?>", "", soup3)    
print(result)    
output=open("output.txt","w")    
output.write(result)    
output.close()    

Answer 1

The following code will get the data from the two tables and output all of it as a json formatted string.

Working Example (Python 2.7.9):

from lxml import html
import requests
import re as regular_expression
import json

page = requests.get("http://www.disastercenter.com/crime/uscrime.htm")
tree = html.fromstring(page.text)

tables = [tree.xpath('//table/tbody/tr[2]/td/center/center/font/table/tbody'),
          tree.xpath('//table/tbody/tr[5]/td/center/center/font/table/tbody')]

tabs = []

for table in tables:
    tab = []
    for row in table:
        for col in row:
            var = col.text_content()
            var = var.strip().replace(" ", "")
            var = var.split('\n')
            if regular_expression.match('^\d{4}$', var[0].strip()):
                tab_row = {}
                tab_row["Year"] = var[0].strip()
                tab_row["Population"] = var[1].strip()
                tab_row["Total"] = var[2].strip()
                tab_row["Violent"] = var[3].strip()
                tab_row["Property"] = var[4].strip()
                tab_row["Murder"] = var[5].strip()
                tab_row["Forcible_Rape"] = var[6].strip()
                tab_row["Robbery"] = var[7].strip()
                tab_row["Aggravated_Assault"] = var[8].strip()
                tab_row["Burglary"] = var[9].strip()
                tab_row["Larceny_Theft"] = var[10].strip()
                tab_row["Vehicle_Theft"] = var[11].strip()
                tab.append(tab_row)
    tabs.append(tab)

json_data = json.dumps(tabs)

output = open("output.txt", "w")
output.write(json_data)
output.close()