intruders

few scripts that checks two letter domains with .mk tld
Log | Files | Refs | README

commit 2cde1d58aa5e26a9c7d4374a6551b580d266a2f0
parent ad662f930f976428f3d9b586d4425760e43a593f
Author: tin <ichtinnotl@gmail.com>
Date:   Sun, 30 Dec 2018 12:57:25 +0100

merges alive.py into intruders.py, refactors the code a little bit

Diffstat:
alive.py | 70----------------------------------------------------------------------
intruders.py | 360++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------
mk_domains.db | 0
requirements.txt | 8++++++++
4 files changed, 283 insertions(+), 155 deletions(-)

diff --git a/alive.py b/alive.py @@ -1,70 +0,0 @@ -import requests, sqlite3 -from bs4 import BeautifulSoup as bs - -def ping_domains(domains, conn): - cur = conn.cursor() - - for domain in domains: - domain = domain[0] - fullDomainName = "http://" + domain - - print(f"Sending a GET request to {domain}...") - - try: - request = requests.get(fullDomainName, timeout=3, allow_redirects=False) - status = request.status_code - title = str(bs(request.content, features="html.parser").title.string) - - query = f"INSERT INTO alive VALUES ('{domain}', '{status}', '{title}', 'YES')" - cur.execute(query) - except Exception as e: - query = f"INSERT INTO alive VALUES ('{domain}', '/', '/', 'NO')" - cur.execute(query) - - print(f"Domain {domain} is not responding: {e} ") - - return None - -def create_connection(db_file): - try: - conn = sqlite3.connect(db_file) - return conn - except Error as e: - print(e) - - return None - -def create_table(conn): - cur = conn.cursor() - - try: - cur.execute('''CREATE TABLE alive (domainName text, status text, title text, alive text)''') - except Exception: - print(e) - - return None - -def fetch_domain_names(conn): - cur = conn.cursor() - cur.execute('SELECT domainName FROM full_log WHERE address="CN"') - - listOfDomains = [] - - for domain in cur.fetchall(): - listOfDomains.append(domain) - - return listOfDomains - -def main(): - database = "two_letter_mk_domains_log.db" - - conn = create_connection(database) - - with conn: - create_table(conn) - domain_names = fetch_domain_names(conn) - ping_domains(domain_names, conn) - - return None - -main() diff --git a/intruders.py b/intruders.py @@ -1,4 +1,6 @@ -import subprocess, pprint, itertools, time, sqlite3 +import subprocess, itertools, time, sqlite3 +import requests +from bs4 import BeautifulSoup as bs addresses = {'MK': 0, 'RS': 0, 'BG': 0, 'GR': 0, 'AL': 0, 'US': 0, 'CA': 0, 'DE': 0, 'FR': 0, 'ES': 0, @@ -10,143 +12,332 @@ alphabet = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'] -def create_database(): - conn = sqlite3.connect('two_letter_mk_domains_log.db') - c = conn.cursor() - +database_saves = 0 + +def create_connection(db_file): + try: + conn = sqlite3.connect(db_file) + return conn + except Error as e: + print(e) + + return None + +def create_tables(conn): + cur = conn.cursor() + try: - c.execute('''CREATE TABLE full_log - (domainName text, address text, registeredDate text, expireDate text)''') + cur.execute('''CREATE TABLE full_log (domain_name text, address text, register_date text, expire_date text)''') - c.execute('''CREATE TABLE clean_log - (address text, totalRegistered int)''') + cur.execute('''CREATE TABLE clean_log (address text, totalRegistered int)''') - c.execute('''CREATE TABLE unregistered_domains_log - (unregistered text)''') + cur.execute('''CREATE TABLE unregistered_domains_log (unregistered text)''') - conn.commit() - conn.close() + cur.execute('''CREATE TABLE alive (domain_name text, status text, title text, alive text)''') + except Exception as e: print(e) return None -def main(): - create_database() +def fetch_domain_names(conn): + cur = conn.cursor() + cur.execute('SELECT domain_name FROM full_log WHERE address="CN"') - countSavesInDatabase = 0 + listOfDomains = [] - for (x, y) in itertools.product(alphabet, repeat = 2): - domain = x + y + ".mk" - command = f"whois {domain} | grep 'domain:\|registered:\|expire:\|address:'" + for domain in cur.fetchall(): + listOfDomains.append(domain) + + return listOfDomains + +def ping_domains(domains, conn): + cur = conn.cursor() + + for domain in domains: + domain = domain[0] + fulldomain_name = "http://" + domain + + print(f"Sending a GET request to {domain}...") try: - output = str((subprocess.check_output(command, shell=True)), 'UTF-8') - except Exception: - addresses["UNREGISTERED_DOMAINS"] += 1 - logUnregisteredDomain(domain) - continue - - domainName, addressName, registeredDate, expireDate = beautifyOutput(output) - saveAsFullLog = writeFullLog(domainName, addressName, registeredDate, expireDate) - - if saveAsFullLog: - countSavesInDatabase += 1 - print(f"Query number {countSavesInDatabase} was successfully written in database.") - else: - print("Cannot write full log to database for some reason.") # 'for some reason' :D + request = requests.get(fulldomain_name, timeout=3, allow_redirects=False) + status = request.status_code + title = str(bs(request.content, features="html.parser").title.string) + + query = f"INSERT INTO alive VALUES ('{domain}', '{status}', '{title}', 'YES')" + cur.execute(query) + except Exception as e: + query = f"INSERT INTO alive VALUES ('{domain}', '/', '/', 'NO')" + cur.execute(query) - # chill or hammer? - # time.sleep(3) + print(f"Domain {domain} is not responding: {e} ") - saveAsCleanLog = writeCleanLog() + return None - if saveAsCleanLog: - print("Clean log was successfully written in database.") - else: - print("Cannot write clean log to database.") +def retreive_domain_data(domain): + command = f"whois {domain} | grep 'domain:\|registered:\|expire:\|address:'" - print("done.") - return True + try : + return str((subprocess.check_output(command, shell=True)), 'UTF-8') + except Exception as e: + return None -def beautifyOutput(output): + +def beautify_output(output): output = output.splitlines() for line in output: - """ Get info from the domain line """ if line.startswith("domain:"): - domainName = line.partition(':')[2].strip() + domain_name = line.partition(':')[2].strip() - """ Get info from the address line """ if line.startswith("address:"): - addressName = line.partition(':')[2].strip() + address_name = line.partition(':')[2].strip() - if addressName in addresses: - addresses[addressName] += 1 + if address_name in addresses: + addresses[address_name] += 1 break - elif len(addressName) == 2 and addressName.isupper(): + elif len(address_name) == 2 and address_name.isupper(): addresses["OTHER_COUNTRY_CODES"] += 1 break else: pass - """ Get info from the registered line """ if line.startswith("registered:"): - registeredDate = line.partition(':')[2].strip() + register_date = line.partition(':')[2].strip() - """ Get info from the expire line """ if line.startswith("expire:"): - expireDate = line.partition(':')[2].strip() - - return domainName, addressName, \ - registeredDate, expireDate + expire_date = line.partition(':')[2].strip() -def writeFullLog(domainName, addressName, registeredDate, expireDate): - conn = sqlite3.connect('two_letter_mk_domains_log.db') - c = conn.cursor() + return domain_name, address_name, register_date, expire_date - query = f"INSERT INTO full_log VALUES ('{domainName}', '{addressName}', '{registeredDate}', '{expireDate}')" - c.execute(query) - conn.commit() - conn.close() +def write_full_data_to_db(domain_name, address_name, register_date, expire_date, conn): + cur = conn.cursor() + query = f"INSERT INTO full_log VALUES ('{domain_name}', '{address_name}', '{register_date}', '{expire_date}')" + global database_saves + try: + cur.execute(query) + except Exception as e: + print(e) + return None + + database_saves += 1 + print(f"{database_saves} {domain_name} was successfully written in the database") return True -def writeCleanLog(): +def write_clean_data_to_db(conn): + cur = conn.cursor() + total_foreign_domains = 0 - skipAddress = ["MK", "UNREGISTERED_DOMAINS"] + skip_address = ["MK", "UNREGISTERED_DOMAINS"] for address in addresses: - if address in skipAddress: + if address in skip_address: pass else: total_foreign_domains += addresses[address] addresses['TOTAL_FOREIGN_DOMAINS'] = total_foreign_domains - conn = sqlite3.connect('two_letter_mk_domains_log.db') - c = conn.cursor() - for address in addresses: - query = f"INSERT INTO clean_log VALUES ('{address}', '{addresses[address]}')" - c.execute(query) - conn.commit() + try: + query = f"INSERT INTO clean_log VALUES ('{address}', '{addresses[address]}')" + cur.execute(query) + print("The clean log was successfully written in the database") + except Exception as e: + print(e) + + return None + +def log_unregistered_domain(domain, conn): + cur = conn.cursor() + global database_saves + + addresses["UNREGISTERED_DOMAINS"] += 1 + + query = f"INSERT INTO unregistered_domains_log VALUES ('{domain}')" + cur.execute(query) - conn.close() - # pprint.pprint(addresses) + database_saves += 1 + print(f"{database_saves} {domain} is UNREGISTERED.") - return True + return None + +def combine_retreive_write(conn): + cur = conn.cursor() + + for (x, y) in itertools.product(alphabet, repeat = 2): + domain = x + y + ".mk" + output = retreive_domain_data(domain) # get data for the domain + + if output: + domain_name, address_name, register_date, expire_date = beautify_output(output) + write_full_data_to_db(domain_name, address_name, register_date, expire_date, conn) + else: + log_unregistered_domain(domain, conn) + + return None + +def main(): + database = "mk_domains.db" + + conn = create_connection(database) + + with conn: + create_tables(conn) + + combine_retreive_write(conn) + domain_names = fetch_domain_names(conn) + ping_domains(domain_names, conn) -def logUnregisteredDomain(unregistered): - conn = sqlite3.connect('two_letter_mk_domains_log.db') - c = conn.cursor() + write_clean_data_to_db(conn) - query = f"INSERT INTO unregistered_domains_log VALUES ('{unregistered}')" - c.execute(query) - conn.commit() - conn.close() + print("done...go on") return None -main()- \ No newline at end of file +main() + + +# def create_database(): +# conn = sqlite3.connect('mk_domains.db') +# c = conn.cursor() + +# try: +# c.execute('''CREATE TABLE full_log +# (domain_name text, address text, register_date text, expire_date text)''') + +# c.execute('''CREATE TABLE clean_log +# (address text, totalRegistered int)''') + +# c.execute('''CREATE TABLE unregistered_domains_log +# (unregistered text)''') + +# conn.commit() +# conn.close() +# except Exception as e: +# print(e) + +# return None + +# def main(): +# create_database() + +# countSavesInDatabase = 0 + +# for (x, y) in itertools.product(alphabet, repeat = 2): +# domain = x + y + ".mk" +# command = f"whois {domain} | grep 'domain:\|registered:\|expire:\|address:'" + +# try: +# output = str((subprocess.check_output(command, shell=True)), 'UTF-8') +# except Exception: +# addresses["UNREGISTERED_DOMAINS"] += 1 +# logUnregisteredDomain(domain) +# continue + +# domain_name, address_name, register_date, expire_date = beautifyOutput(output) +# saveAsFullLog = writeFullLog(domain_name, address_name, register_date, expire_date) + +# if saveAsFullLog: +# countSavesInDatabase += 1 +# print(f"Query number {countSavesInDatabase} was successfully written in database.") +# else: +# print("Cannot write full log to database for some reason.") # 'for some reason' :D + +# # chill or hammer? +# # time.sleep(3) + +# saveAsCleanLog = writeCleanLog() + +# if saveAsCleanLog: +# print("Clean log was successfully written in database.") +# else: +# print("Cannot write clean log to database.") + +# print("done.") +# return True + +# def beautifyOutput(output): +# output = output.splitlines() + +# for line in output: +# """ Get info from the domain line """ +# if line.startswith("domain:"): +# domain_name = line.partition(':')[2].strip() + +# """ Get info from the address line """ +# if line.startswith("address:"): +# address_name = line.partition(':')[2].strip() + +# if address_name in addresses: +# addresses[address_name] += 1 +# break +# elif len(address_name) == 2 and address_name.isupper(): +# addresses["OTHER_COUNTRY_CODES"] += 1 +# break +# else: +# pass + +# """ Get info from the registered line """ +# if line.startswith("registered:"): +# register_date = line.partition(':')[2].strip() + +# """ Get info from the expire line """ +# if line.startswith("expire:"): +# expire_date = line.partition(':')[2].strip() + +# return domain_name, address_name, \ +# register_date, expire_date + +# def writeFullLog(domain_name, address_name, register_date, expire_date): +# conn = sqlite3.connect('two_letter_mk_domains_log.db') +# c = conn.cursor() + +# query = f"INSERT INTO full_log VALUES ('{domain_name}', '{address_name}', '{register_date}', '{expire_date}')" +# c.execute(query) +# conn.commit() +# conn.close() + +# return True + +# def writeCleanLog(): +# total_foreign_domains = 0 +# skipAddress = ["MK", "UNREGISTERED_DOMAINS"] + +# for address in addresses: +# if address in skipAddress: +# pass +# else: +# total_foreign_domains += addresses[address] + +# addresses['TOTAL_FOREIGN_DOMAINS'] = total_foreign_domains + +# conn = sqlite3.connect('two_letter_mk_domains_log.db') +# c = conn.cursor() + +# for address in addresses: +# query = f"INSERT INTO clean_log VALUES ('{address}', '{addresses[address]}')" +# c.execute(query) +# conn.commit() + +# conn.close() +# # pprint.pprint(addresses) + +# return True + + +# def logUnregisteredDomain(unregistered): +# conn = sqlite3.connect('two_letter_mk_domains_log.db') +# c = conn.cursor() + +# query = f"INSERT INTO unregistered_domains_log VALUES ('{unregistered}')" +# c.execute(query) +# conn.commit() +# conn.close() + +# return None + +# main() diff --git a/mk_domains.db b/mk_domains.db Binary files differ. diff --git a/requirements.txt b/requirements.txt @@ -0,0 +1,8 @@ +beautifulsoup4==4.6.3 +bs4==0.0.1 +certifi==2018.11.29 +chardet==3.0.4 +dnspython==1.16.0 +idna==2.8 +requests==2.21.0 +urllib3==1.24.1