Python code for scrapping Nrega data

from selenium import webdriver
import os
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains

browser = webdriver.Firefox()
fp = webdriver.FirefoxProfile()
fp.set_preference("browser.download.manager.showWhenStarting",False)
fp.set_preference("browser.helperApps.neverAsk.saveToDisk","html")

for x in range(10,11):

	# url = 'http://164.100.129.6/netnrega/app_issue.aspx?page=s&lflag=&state_code='+str(x)+'&fin_year=2014-2015'
	url='http://nrega.nic.in/netnrega/homestciti.aspx?state_code=13&state_name=HIMACHAL%20PRADESH'
	browser.get(url)
	browser.find_element_by_link_text('BILASPUR').click()
	# browser.find_element_by_link_text('Bilaspur Sadar').click()

	BlockTable=browser.find_element_by_id('gvdpc')
	BlockLinks=BlockTable.find_elements_by_tag_name('a')
	for k in range(1,2):
		BlockLinks[k].send_keys(Keys.COMMAND +Keys.RETURN)
		browser.find_element_by_tag_name('body').send_keys(Keys.COMMAND + Keys.ALT + Keys.RIGHT)
		browser.switch_to_window(browser.window_handles[-1])

		PanchayatTable=browser.find_element_by_id('ctl00_ContentPlaceHolder1_gvpanch')
		PanchayatLinks=PanchayatTable.find_elements_by_tag_name('a')
		for j in range(39,40):
			x=46
			# if j==10:
			# 	x=160
			# else:
			# 	x=1
			while True:
				PanchayatLinks[j].send_keys(Keys.COMMAND + Keys.RETURN)

				browser.find_element_by_tag_name('body').send_keys(Keys.COMMAND + Keys.ALT + Keys.RIGHT)
				browser.switch_to_window(browser.window_handles[-1])

				browser.find_element_by_link_text('Job card/Employment Register').click()

				links=browser.find_elements_by_tag_name('a')
				
				for i in range(x,len(links)):	
					x=x+1
					links[i].send_keys(Keys.COMMAND + Keys.RETURN)
					browser.find_element_by_tag_name('body').send_keys(Keys.COMMAND + Keys.ALT + Keys.RIGHT)
					
					time.sleep(1)
					browser.switch_to_window(browser.window_handles[-1])
					html_source = browser.page_source
					# Open a file
					fo = open("/Users/abhiuser7/Desktop/NREGA/down2/"+str(k)+"-"+str(j)+"-"+str(i)+".html", "wb")
					fo.write( html_source.encode('utf8'));
					fo.close()
					# time.sleep(0.5)
					# element = WebDriverWait(browser, 10).until(
					# 	EC.presence_of_element_located((By.ID, "lbl_familyid"))
			#       )
					browser.find_element_by_tag_name('body').send_keys(Keys.COMMAND + 'w')
					browser.switch_to_window(browser.window_handles[-1])
					if x%150 ==0:
						break

				browser.find_element_by_tag_name('body').send_keys(Keys.COMMAND + 'w')
				browser.switch_to_window(browser.window_handles[-1])
				if x==len(links):
					break
		browser.find_element_by_tag_name('body').send_keys(Keys.COMMAND + 'w')
		browser.switch_to_window(browser.window_handles[-1])
		# browser.switch_to_window(main_window)

		# links2=browser.find_elements_by_tag_name('a')
		# links2[0].click()


	
	# browser.find_element_by_id('ctl00_ContentPlaceHolder1_Rbtnlist_1').click()	
	# browser.find_element_by_id('ctl00_ContentPlaceHolder1_LinkButton1').click()
	# os.chdir("/Users/abhiuser7/Downloads/")
	# files = filter(os.path.isfile, os.listdir("/Users/abhiuser7/Downloads/"))
	# files = [os.path.join("/Users/abhiuser7/Downloads/", f) for f in files]
	# files.sort(key=lambda x: os.path.getmtime(x))
	# newest_file = files[-1]
	# os.rename(newest_file, str(x)+".xls")