Updated File for fetching

This commit is contained in:
Christian Risi 2025-09-18 17:23:56 +02:00
parent 9838e287a4
commit 7f0c5ce8d3

View File

@ -1,3 +1,4 @@
from math import floor
from time import sleep
import SPARQLWrapper
import requests
@ -8,9 +9,11 @@ TYPE = SPARQLWrapper.CSV
TIMEOUT_SECONDS = 1.5
LIMIT = int(1E4)
OFFSET = LIMIT
INITIAL_OFFSET = 0
INITIAL_OFFSET = 15200000
MAX_PAGES = int(1E9)
# Missing page 13220000
FILE_URI = "./Assets/Dataset/1-hop/dataset.csv"
QUERY = """
@ -35,13 +38,13 @@ WHERE {
def main():
exit = False
page = INITIAL_OFFSET
page = int(floor(INITIAL_OFFSET / LIMIT)) -1
while not exit:
print(f"Starting to get page {page}")
CURRENT_OFFSET = OFFSET * page
CURRENT_OFFSET = int(OFFSET + (page * LIMIT))
sparql = SPARQLWrapper.SPARQLWrapper(BASE_URL)
sparql.setReturnFormat(TYPE)