80 lines
1.8 KiB
Python
80 lines
1.8 KiB
Python
from time import sleep
|
|
import SPARQLWrapper
|
|
import requests
|
|
|
|
|
|
BASE_URL = "https://dbpedia.org/sparql"
|
|
TYPE = SPARQLWrapper.CSV
|
|
TIMEOUT_SECONDS = int(10)
|
|
LIMIT = int(1E6)
|
|
OFFSET = LIMIT
|
|
INITIAL_OFFSET = 0
|
|
MAX_PAGES = int(1E2)
|
|
|
|
FILE_URI = "./Assets/Dataset/1-hop/dataset.csv"
|
|
|
|
QUERY = """
|
|
PREFIX dbo: <http://dbpedia.org/ontology/>
|
|
PREFIX dbp: <http://dbpedia.org/property/>
|
|
PREFIX dbr: <http://dbpedia.org/resource/>
|
|
|
|
SELECT ?subject, ?relationship, ?object
|
|
WHERE {
|
|
?subject ?relationship ?object .
|
|
?object rdf:type dbo:Film .
|
|
FILTER (?relationship != <http://dbpedia.org/ontology/wikiPageWikiLink>)
|
|
}"""
|
|
|
|
|
|
|
|
def main():
|
|
|
|
for page in range(INITIAL_OFFSET, MAX_PAGES):
|
|
|
|
print(f"Starting to get page {page}")
|
|
|
|
CURRENT_OFFSET = OFFSET * page
|
|
sparql = SPARQLWrapper.SPARQLWrapper(BASE_URL)
|
|
|
|
sparql.setReturnFormat(TYPE)
|
|
|
|
CURRENT_PAGE_QUERY = "\n".join([
|
|
QUERY,
|
|
f"LIMIT {LIMIT}",
|
|
f"OFFSET {CURRENT_OFFSET}"
|
|
])
|
|
|
|
print(f"\nCurrent Query:\n{CURRENT_PAGE_QUERY}\n")
|
|
|
|
sparql.setQuery(CURRENT_PAGE_QUERY)
|
|
|
|
try:
|
|
res = sparql.queryAndConvert()
|
|
text = ""
|
|
|
|
if type(res) == bytes:
|
|
|
|
initial_offset = 0
|
|
|
|
if page != 0:
|
|
initial_offset = 1
|
|
|
|
lines = res.decode("utf-8", "ignore").split("\n")
|
|
text = "\n".join(lines[initial_offset:])
|
|
|
|
with open(FILE_URI, "a+", encoding="utf-8") as dataset:
|
|
|
|
print(f"Writing page {page} on {FILE_URI}")
|
|
|
|
dataset.write(
|
|
text
|
|
)
|
|
|
|
except Exception as ex:
|
|
print(f"Something went wrong during page {page}:\n\t{ex}")
|
|
|
|
print(f"Sleeping for {TIMEOUT_SECONDS}")
|
|
sleep(TIMEOUT_SECONDS)
|
|
|
|
main()
|