PoliBa-DeepLearning/NanoSocrates
2
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Updated File for fetching

Browse Source
This commit is contained in:
Christian Risi 2025-09-18 17:23:56 +02:00
parent 9838e287a4
commit 7f0c5ce8d3
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Script/fetchdata.py
View File

@ -1,3 +1,4 @@
from math import floor
from time import sleep from time import sleep
import SPARQLWrapper import SPARQLWrapper
import requests import requests
@ -8,9 +9,11 @@ TYPE = SPARQLWrapper.CSV
TIMEOUT_SECONDS = 1.5 TIMEOUT_SECONDS = 1.5
LIMIT = int(1E4) LIMIT = int(1E4)
OFFSET = LIMIT OFFSET = LIMIT
INITIAL_OFFSET = 0 INITIAL_OFFSET = 15200000
MAX_PAGES = int(1E9) MAX_PAGES = int(1E9)
# Missing page 13220000
FILE_URI = "./Assets/Dataset/1-hop/dataset.csv" FILE_URI = "./Assets/Dataset/1-hop/dataset.csv"
QUERY = """ QUERY = """
@ -35,13 +38,13 @@ WHERE {
def main(): def main():
exit = False exit = False
page = INITIAL_OFFSET page = int(floor(INITIAL_OFFSET / LIMIT)) -1
while not exit: while not exit:
print(f"Starting to get page {page}") print(f"Starting to get page {page}")
CURRENT_OFFSET = OFFSET * page CURRENT_OFFSET = int(OFFSET + (page * LIMIT))
sparql = SPARQLWrapper.SPARQLWrapper(BASE_URL) sparql = SPARQLWrapper.SPARQLWrapper(BASE_URL)
sparql.setReturnFormat(TYPE) sparql.setReturnFormat(TYPE)